Mercurial > hg > graal-compiler

changeset 651:8ce995316d10

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge
author acorn
date Mon, 16 Mar 2009 08:50:53 -0400
parents 2581d90c6c9b (diff) 54782a4cd321 (current diff)
children 4aaa9f5e02a8
files
diffstat 225 files changed, 2467 insertions(+), 982 deletions(-) [+]
line wrap: on
line diff
--- a/.hgtags	Sun Mar 15 18:11:46 2009 -0700
+++ b/.hgtags	Mon Mar 16 08:50:53 2009 -0400
@@ -22,3 +22,6 @@
 945bf754069766e76873c53102fae48abf04cf5b jdk7-b45
 16bb38eeda35b46268eefa4c1f829eb086e0ca46 jdk7-b46
 fcb923bad68e2b10380a030ea83a723f4dc3d4d6 jdk7-b47
+bcb33806d186561c781992e5f4d8a90bb033f9f0 jdk7-b48
+8b22ccb5aba2c6c11bddf6488a7bb7ef5b4bf2be jdk7-b49
+dae503d9f04c1a11e182dbf7f770509c28dc0609 jdk7-b50
--- a/agent/src/os/linux/ps_core.c	Sun Mar 15 18:11:46 2009 -0700
+++ b/agent/src/os/linux/ps_core.c	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/agent/src/os/solaris/proc/saproc.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/agent/src/os/solaris/proc/saproc.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/make/hotspot_version	Sun Mar 15 18:11:46 2009 -0700
+++ b/make/hotspot_version	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 # 
-# Copyright 2006-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 2006-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -35,7 +35,7 @@
 
 HS_MAJOR_VER=15
 HS_MINOR_VER=0
-HS_BUILD_NUMBER=02
+HS_BUILD_NUMBER=03
 
 JDK_MAJOR_VER=1
 JDK_MINOR_VER=7
--- a/make/linux/makefiles/gcc.make	Sun Mar 15 18:11:46 2009 -0700
+++ b/make/linux/makefiles/gcc.make	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 #
-# Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/sparc/vm/assembler_sparc.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/sparc/vm/assembler_sparc.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -2615,6 +2615,158 @@
   }
 }
 
+RegisterConstant MacroAssembler::delayed_value(intptr_t* delayed_value_addr,
+                                               Register tmp,
+                                               int offset) {
+  intptr_t value = *delayed_value_addr;
+  if (value != 0)
+    return RegisterConstant(value + offset);
+
+  // load indirectly to solve generation ordering problem
+  Address a(tmp, (address) delayed_value_addr);
+  load_ptr_contents(a, tmp);
+
+#ifdef ASSERT
+  tst(tmp);
+  breakpoint_trap(zero, xcc);
+#endif
+
+  if (offset != 0)
+    add(tmp, offset, tmp);
+
+  return RegisterConstant(tmp);
+}
+
+
+void MacroAssembler::regcon_inc_ptr( RegisterConstant& dest, RegisterConstant src, Register temp ) {
+  assert(dest.register_or_noreg() != G0, "lost side effect");
+  if ((src.is_constant() && src.as_constant() == 0) ||
+      (src.is_register() && src.as_register() == G0)) {
+    // do nothing
+  } else if (dest.is_register()) {
+    add(dest.as_register(), ensure_rs2(src, temp), dest.as_register());
+  } else if (src.is_constant()) {
+    intptr_t res = dest.as_constant() + src.as_constant();
+    dest = RegisterConstant(res); // side effect seen by caller
+  } else {
+    assert(temp != noreg, "cannot handle constant += register");
+    add(src.as_register(), ensure_rs2(dest, temp), temp);
+    dest = RegisterConstant(temp); // side effect seen by caller
+  }
+}
+
+void MacroAssembler::regcon_sll_ptr( RegisterConstant& dest, RegisterConstant src, Register temp ) {
+  assert(dest.register_or_noreg() != G0, "lost side effect");
+  if (!is_simm13(src.constant_or_zero()))
+    src = (src.as_constant() & 0xFF);
+  if ((src.is_constant() && src.as_constant() == 0) ||
+      (src.is_register() && src.as_register() == G0)) {
+    // do nothing
+  } else if (dest.is_register()) {
+    sll_ptr(dest.as_register(), src, dest.as_register());
+  } else if (src.is_constant()) {
+    intptr_t res = dest.as_constant() << src.as_constant();
+    dest = RegisterConstant(res); // side effect seen by caller
+  } else {
+    assert(temp != noreg, "cannot handle constant <<= register");
+    set(dest.as_constant(), temp);
+    sll_ptr(temp, src, temp);
+    dest = RegisterConstant(temp); // side effect seen by caller
+  }
+}
+
+
+// Look up the method for a megamorphic invokeinterface call.
+// The target method is determined by <intf_klass, itable_index>.
+// The receiver klass is in recv_klass.
+// On success, the result will be in method_result, and execution falls through.
+// On failure, execution transfers to the given label.
+void MacroAssembler::lookup_interface_method(Register recv_klass,
+                                             Register intf_klass,
+                                             RegisterConstant itable_index,
+                                             Register method_result,
+                                             Register scan_temp,
+                                             Register sethi_temp,
+                                             Label& L_no_such_interface) {
+  assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
+  assert(itable_index.is_constant() || itable_index.as_register() == method_result,
+         "caller must use same register for non-constant itable index as for method");
+
+  // Compute start of first itableOffsetEntry (which is at the end of the vtable)
+  int vtable_base = instanceKlass::vtable_start_offset() * wordSize;
+  int scan_step   = itableOffsetEntry::size() * wordSize;
+  int vte_size    = vtableEntry::size() * wordSize;
+
+  lduw(recv_klass, instanceKlass::vtable_length_offset() * wordSize, scan_temp);
+  // %%% We should store the aligned, prescaled offset in the klassoop.
+  // Then the next several instructions would fold away.
+
+  int round_to_unit = ((HeapWordsPerLong > 1) ? BytesPerLong : 0);
+  int itb_offset = vtable_base;
+  if (round_to_unit != 0) {
+    // hoist first instruction of round_to(scan_temp, BytesPerLong):
+    itb_offset += round_to_unit - wordSize;
+  }
+  int itb_scale = exact_log2(vtableEntry::size() * wordSize);
+  sll(scan_temp, itb_scale,  scan_temp);
+  add(scan_temp, itb_offset, scan_temp);
+  if (round_to_unit != 0) {
+    // Round up to align_object_offset boundary
+    // see code for instanceKlass::start_of_itable!
+    // Was: round_to(scan_temp, BytesPerLong);
+    // Hoisted: add(scan_temp, BytesPerLong-1, scan_temp);
+    and3(scan_temp, -round_to_unit, scan_temp);
+  }
+  add(recv_klass, scan_temp, scan_temp);
+
+  // Adjust recv_klass by scaled itable_index, so we can free itable_index.
+  RegisterConstant itable_offset = itable_index;
+  regcon_sll_ptr(itable_offset, exact_log2(itableMethodEntry::size() * wordSize));
+  regcon_inc_ptr(itable_offset, itableMethodEntry::method_offset_in_bytes());
+  add(recv_klass, ensure_rs2(itable_offset, sethi_temp), recv_klass);
+
+  // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
+  //   if (scan->interface() == intf) {
+  //     result = (klass + scan->offset() + itable_index);
+  //   }
+  // }
+  Label search, found_method;
+
+  for (int peel = 1; peel >= 0; peel--) {
+    // %%%% Could load both offset and interface in one ldx, if they were
+    // in the opposite order.  This would save a load.
+    ld_ptr(scan_temp, itableOffsetEntry::interface_offset_in_bytes(), method_result);
+
+    // Check that this entry is non-null.  A null entry means that
+    // the receiver class doesn't implement the interface, and wasn't the
+    // same as when the caller was compiled.
+    bpr(Assembler::rc_z, false, Assembler::pn, method_result, L_no_such_interface);
+    delayed()->cmp(method_result, intf_klass);
+
+    if (peel) {
+      brx(Assembler::equal,    false, Assembler::pt, found_method);
+    } else {
+      brx(Assembler::notEqual, false, Assembler::pn, search);
+      // (invert the test to fall through to found_method...)
+    }
+    delayed()->add(scan_temp, scan_step, scan_temp);
+
+    if (!peel)  break;
+
+    bind(search);
+  }
+
+  bind(found_method);
+
+  // Got a hit.
+  int ito_offset = itableOffsetEntry::offset_offset_in_bytes();
+  // scan_temp[-scan_step] points to the vtable offset we need
+  ito_offset -= scan_step;
+  lduw(scan_temp, ito_offset, scan_temp);
+  ld_ptr(recv_klass, scan_temp, method_result);
+}
+
+
 void MacroAssembler::biased_locking_enter(Register obj_reg, Register mark_reg,
                                           Register temp_reg,
                                           Label& done, Label* slow_case,
@@ -4057,6 +4209,24 @@
   card_table_write(bs->byte_map_base, tmp, store_addr);
 }
 
+// Loading values by size and signed-ness
+void MacroAssembler::load_sized_value(Register s1, RegisterConstant s2, Register d,
+                                      int size_in_bytes, bool is_signed) {
+  switch (size_in_bytes ^ (is_signed ? -1 : 0)) {
+  case ~8:  // fall through:
+  case  8:  ld_long( s1, s2, d ); break;
+  case ~4:  ldsw(    s1, s2, d ); break;
+  case  4:  lduw(    s1, s2, d ); break;
+  case ~2:  ldsh(    s1, s2, d ); break;
+  case  2:  lduh(    s1, s2, d ); break;
+  case ~1:  ldsb(    s1, s2, d ); break;
+  case  1:  ldub(    s1, s2, d ); break;
+  default:  ShouldNotReachHere();
+  }
+}
+
+
+
 void MacroAssembler::load_klass(Register src_oop, Register klass) {
   // The number of bytes in this code is used by
   // MachCallDynamicJavaNode::ret_addr_offset()
--- a/src/cpu/sparc/vm/assembler_sparc.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/sparc/vm/assembler_sparc.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -384,6 +384,12 @@
 
   inline bool is_simm13(int offset = 0);  // check disp+offset for overflow
 
+  Address plus_disp(int disp) const {     // bump disp by a small amount
+    Address a = (*this);
+    a._disp += disp;
+    return a;
+  }
+
   Address split_disp() const {            // deal with disp overflow
     Address a = (*this);
     int hi_disp = _disp & ~0x3ff;
@@ -1082,6 +1088,7 @@
   inline void add(    Register s1, Register s2, Register d );
   inline void add(    Register s1, int simm13a, Register d, relocInfo::relocType rtype = relocInfo::none);
   inline void add(    Register s1, int simm13a, Register d, RelocationHolder const& rspec);
+  inline void add(    Register s1, RegisterConstant s2, Register d, int offset = 0);
   inline void add(    const Address&  a,              Register d, int offset = 0);
 
   void addcc(  Register s1, Register s2, Register d ) { emit_long( op(arith_op) | rd(d) | op3(add_op3  | cc_bit_op3) | rs1(s1) | rs2(s2) ); }
@@ -1298,6 +1305,16 @@
   inline void ld(   const Address& a, Register d, int offset = 0 );
   inline void ldd(  const Address& a, Register d, int offset = 0 );
 
+  inline void ldub(  Register s1, RegisterConstant s2, Register d );
+  inline void ldsb(  Register s1, RegisterConstant s2, Register d );
+  inline void lduh(  Register s1, RegisterConstant s2, Register d );
+  inline void ldsh(  Register s1, RegisterConstant s2, Register d );
+  inline void lduw(  Register s1, RegisterConstant s2, Register d );
+  inline void ldsw(  Register s1, RegisterConstant s2, Register d );
+  inline void ldx(   Register s1, RegisterConstant s2, Register d );
+  inline void ld(    Register s1, RegisterConstant s2, Register d );
+  inline void ldd(   Register s1, RegisterConstant s2, Register d );
+
   // pp 177
 
   void ldsba(  Register s1, Register s2, int ia, Register d ) {             emit_long( op(ldst_op) | rd(d) | op3(ldsb_op3 | alt_bit_op3) | rs1(s1) | imm_asi(ia) | rs2(s2) ); }
@@ -1518,6 +1535,13 @@
   inline void st(   Register d, const Address& a, int offset = 0 );
   inline void std(  Register d, const Address& a, int offset = 0 );
 
+  inline void stb(  Register d, Register s1, RegisterConstant s2 );
+  inline void sth(  Register d, Register s1, RegisterConstant s2 );
+  inline void stw(  Register d, Register s1, RegisterConstant s2 );
+  inline void stx(  Register d, Register s1, RegisterConstant s2 );
+  inline void std(  Register d, Register s1, RegisterConstant s2 );
+  inline void st(   Register d, Register s1, RegisterConstant s2 );
+
   // pp 177
 
   void stba(  Register d, Register s1, Register s2, int ia ) {             emit_long( op(ldst_op) | rd(d) | op3(stb_op3 | alt_bit_op3) | rs1(s1) | imm_asi(ia) | rs2(s2) ); }
@@ -1835,6 +1859,7 @@
   // Functions for isolating 64 bit shifts for LP64
   inline void sll_ptr( Register s1, Register s2, Register d );
   inline void sll_ptr( Register s1, int imm6a,   Register d );
+  inline void sll_ptr( Register s1, RegisterConstant s2, Register d );
   inline void srl_ptr( Register s1, Register s2, Register d );
   inline void srl_ptr( Register s1, int imm6a,   Register d );
 
@@ -1940,20 +1965,47 @@
   // st_ptr will perform st for 32 bit VM's and stx for 64 bit VM's
   inline void ld_ptr(   Register s1, Register s2, Register d );
   inline void ld_ptr(   Register s1, int simm13a, Register d);
+  inline void ld_ptr(   Register s1, RegisterConstant s2, Register d );
   inline void ld_ptr(  const Address& a, Register d, int offset = 0 );
   inline void st_ptr(  Register d, Register s1, Register s2 );
   inline void st_ptr(  Register d, Register s1, int simm13a);
+  inline void st_ptr(  Register d, Register s1, RegisterConstant s2 );
   inline void st_ptr(  Register d, const Address& a, int offset = 0 );
 
   // ld_long will perform ld for 32 bit VM's and ldx for 64 bit VM's
   // st_long will perform st for 32 bit VM's and stx for 64 bit VM's
   inline void ld_long( Register s1, Register s2, Register d );
   inline void ld_long( Register s1, int simm13a, Register d );
+  inline void ld_long( Register s1, RegisterConstant s2, Register d );
   inline void ld_long( const Address& a, Register d, int offset = 0 );
   inline void st_long( Register d, Register s1, Register s2 );
   inline void st_long( Register d, Register s1, int simm13a );
+  inline void st_long( Register d, Register s1, RegisterConstant s2 );
   inline void st_long( Register d, const Address& a, int offset = 0 );
 
+  // Loading values by size and signed-ness
+  void load_sized_value(Register s1, RegisterConstant s2, Register d,
+                        int size_in_bytes, bool is_signed);
+
+  // Helpers for address formation.
+  // They update the dest in place, whether it is a register or constant.
+  // They emit no code at all if src is a constant zero.
+  // If dest is a constant and src is a register, the temp argument
+  // is required, and becomes the result.
+  // If dest is a register and src is a non-simm13 constant,
+  // the temp argument is required, and is used to materialize the constant.
+  void regcon_inc_ptr( RegisterConstant& dest, RegisterConstant src,
+                       Register temp = noreg );
+  void regcon_sll_ptr( RegisterConstant& dest, RegisterConstant src,
+                       Register temp = noreg );
+  RegisterConstant ensure_rs2(RegisterConstant rs2, Register sethi_temp) {
+    guarantee(sethi_temp != noreg, "constant offset overflow");
+    if (is_simm13(rs2.constant_or_zero()))
+      return rs2;               // register or short constant
+    set(rs2.as_constant(), sethi_temp);
+    return sethi_temp;
+  }
+
   // --------------------------------------------------
 
  public:
@@ -2267,6 +2319,14 @@
   );
   void tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case);
 
+  // interface method calling
+  void lookup_interface_method(Register recv_klass,
+                               Register intf_klass,
+                               RegisterConstant itable_index,
+                               Register method_result,
+                               Register temp_reg, Register temp2_reg,
+                               Label& no_such_interface);
+
   // Stack overflow checking
 
   // Note: this clobbers G3_scratch
@@ -2281,6 +2341,8 @@
   // stack overflow + shadow pages.  Clobbers tsp and scratch registers.
   void bang_stack_size(Register Rsize, Register Rtsp, Register Rscratch);
 
+  virtual RegisterConstant delayed_value(intptr_t* delayed_value_addr, Register tmp, int offset);
+
   void verify_tlab();
 
   Condition negate_condition(Condition cond);
--- a/src/cpu/sparc/vm/assembler_sparc.inline.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/sparc/vm/assembler_sparc.inline.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -143,6 +143,49 @@
 inline void Assembler::ld(  Register s1, int simm13a, Register d) { lduw( s1, simm13a, d); }
 #endif
 
+inline void Assembler::ldub(  Register s1, RegisterConstant s2, Register d) {
+  if (s2.is_register())  ldsb(s1, s2.as_register(), d);
+  else                   ldsb(s1, s2.as_constant(), d);
+}
+inline void Assembler::ldsb(  Register s1, RegisterConstant s2, Register d) {
+  if (s2.is_register())  ldsb(s1, s2.as_register(), d);
+  else                   ldsb(s1, s2.as_constant(), d);
+}
+inline void Assembler::lduh(  Register s1, RegisterConstant s2, Register d) {
+  if (s2.is_register())  ldsh(s1, s2.as_register(), d);
+  else                   ldsh(s1, s2.as_constant(), d);
+}
+inline void Assembler::ldsh(  Register s1, RegisterConstant s2, Register d) {
+  if (s2.is_register())  ldsh(s1, s2.as_register(), d);
+  else                   ldsh(s1, s2.as_constant(), d);
+}
+inline void Assembler::lduw(  Register s1, RegisterConstant s2, Register d) {
+  if (s2.is_register())  ldsw(s1, s2.as_register(), d);
+  else                   ldsw(s1, s2.as_constant(), d);
+}
+inline void Assembler::ldsw(  Register s1, RegisterConstant s2, Register d) {
+  if (s2.is_register())  ldsw(s1, s2.as_register(), d);
+  else                   ldsw(s1, s2.as_constant(), d);
+}
+inline void Assembler::ldx(   Register s1, RegisterConstant s2, Register d) {
+  if (s2.is_register())  ldx(s1, s2.as_register(), d);
+  else                   ldx(s1, s2.as_constant(), d);
+}
+inline void Assembler::ld(    Register s1, RegisterConstant s2, Register d) {
+  if (s2.is_register())  ld(s1, s2.as_register(), d);
+  else                   ld(s1, s2.as_constant(), d);
+}
+inline void Assembler::ldd(   Register s1, RegisterConstant s2, Register d) {
+  if (s2.is_register())  ldd(s1, s2.as_register(), d);
+  else                   ldd(s1, s2.as_constant(), d);
+}
+
+// form effective addresses this way:
+inline void Assembler::add(   Register s1, RegisterConstant s2, Register d, int offset) {
+  if (s2.is_register())  add(s1, s2.as_register(), d);
+  else                 { add(s1, s2.as_constant() + offset, d); offset = 0; }
+  if (offset != 0)       add(d,  offset,                    d);
+}
 
 inline void Assembler::ld(   const Address& a, Register d, int offset ) { relocate(a.rspec(offset)); ld(   a.base(), a.disp() + offset, d ); }
 inline void Assembler::ldsb( const Address& a, Register d, int offset ) { relocate(a.rspec(offset)); ldsb( a.base(), a.disp() + offset, d ); }
@@ -200,6 +243,27 @@
 inline void Assembler::st(  Register d, Register s1, Register s2) { stw(d, s1, s2); }
 inline void Assembler::st(  Register d, Register s1, int simm13a) { stw(d, s1, simm13a); }
 
+inline void Assembler::stb(  Register d, Register s1, RegisterConstant s2) {
+  if (s2.is_register())  stb(d, s1, s2.as_register());
+  else                   stb(d, s1, s2.as_constant());
+}
+inline void Assembler::sth(  Register d, Register s1, RegisterConstant s2) {
+  if (s2.is_register())  sth(d, s1, s2.as_register());
+  else                   sth(d, s1, s2.as_constant());
+}
+inline void Assembler::stx(  Register d, Register s1, RegisterConstant s2) {
+  if (s2.is_register())  stx(d, s1, s2.as_register());
+  else                   stx(d, s1, s2.as_constant());
+}
+inline void Assembler::std( Register d, Register s1, RegisterConstant s2) {
+  if (s2.is_register())  std(d, s1, s2.as_register());
+  else                   std(d, s1, s2.as_constant());
+}
+inline void Assembler::st(  Register d, Register s1, RegisterConstant s2) {
+  if (s2.is_register())  st(d, s1, s2.as_register());
+  else                   st(d, s1, s2.as_constant());
+}
+
 inline void Assembler::stb( Register d, const Address& a, int offset) { relocate(a.rspec(offset)); stb( d, a.base(), a.disp() + offset); }
 inline void Assembler::sth( Register d, const Address& a, int offset) { relocate(a.rspec(offset)); sth( d, a.base(), a.disp() + offset); }
 inline void Assembler::stw( Register d, const Address& a, int offset) { relocate(a.rspec(offset)); stw( d, a.base(), a.disp() + offset); }
@@ -244,6 +308,14 @@
 #endif
 }
 
+inline void MacroAssembler::ld_ptr( Register s1, RegisterConstant s2, Register d ) {
+#ifdef _LP64
+  Assembler::ldx( s1, s2, d);
+#else
+  Assembler::ld(  s1, s2, d);
+#endif
+}
+
 inline void MacroAssembler::ld_ptr( const Address& a, Register d, int offset ) {
 #ifdef _LP64
   Assembler::ldx(  a, d, offset );
@@ -268,6 +340,14 @@
 #endif
 }
 
+inline void MacroAssembler::st_ptr( Register d, Register s1, RegisterConstant s2 ) {
+#ifdef _LP64
+  Assembler::stx( d, s1, s2);
+#else
+  Assembler::st( d, s1, s2);
+#endif
+}
+
 inline void MacroAssembler::st_ptr(  Register d, const Address& a, int offset) {
 #ifdef _LP64
   Assembler::stx(  d, a, offset);
@@ -293,6 +373,14 @@
 #endif
 }
 
+inline void MacroAssembler::ld_long( Register s1, RegisterConstant s2, Register d ) {
+#ifdef _LP64
+  Assembler::ldx(s1, s2, d);
+#else
+  Assembler::ldd(s1, s2, d);
+#endif
+}
+
 inline void MacroAssembler::ld_long( const Address& a, Register d, int offset ) {
 #ifdef _LP64
   Assembler::ldx(a, d, offset );
@@ -317,6 +405,14 @@
 #endif
 }
 
+inline void MacroAssembler::st_long( Register d, Register s1, RegisterConstant s2 ) {
+#ifdef _LP64
+  Assembler::stx(d, s1, s2);
+#else
+  Assembler::std(d, s1, s2);
+#endif
+}
+
 inline void MacroAssembler::st_long( Register d, const Address& a, int offset ) {
 #ifdef _LP64
   Assembler::stx(d, a, offset);
@@ -359,6 +455,11 @@
 #endif
 }
 
+inline void MacroAssembler::sll_ptr( Register s1, RegisterConstant s2, Register d ) {
+  if (s2.is_register())  sll_ptr(s1, s2.as_register(), d);
+  else                   sll_ptr(s1, s2.as_constant(), d);
+}
+
 // Use the right branch for the platform
 
 inline void MacroAssembler::br( Condition c, bool a, Predict p, address d, relocInfo::relocType rt ) {
--- a/src/cpu/sparc/vm/jni_sparc.h	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/sparc/vm/jni_sparc.h	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/sparc/vm/sparc.ad	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/sparc/vm/sparc.ad	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -5286,55 +5286,91 @@
   ins_cost(MEMORY_REF_COST);
 
   size(4);
-  format %{ "LDSB   $mem,$dst" %}
+  format %{ "LDSB   $mem,$dst\t! byte" %}
+  opcode(Assembler::ldsb_op3);
+  ins_encode(simple_form3_mem_reg( mem, dst ) );
+  ins_pipe(iload_mask_mem);
+%}
+
+// Load Byte (8bit signed) into a Long Register
+instruct loadB2L(iRegL dst, memory mem) %{
+  match(Set dst (ConvI2L (LoadB mem)));
+  ins_cost(MEMORY_REF_COST);
+
+  size(4);
+  format %{ "LDSB   $mem,$dst\t! byte -> long" %}
   opcode(Assembler::ldsb_op3);
   ins_encode(simple_form3_mem_reg( mem, dst ) );
   ins_pipe(iload_mask_mem);
 %}
 
-// Load Byte (8bit UNsigned) into an int reg
-instruct loadUB(iRegI dst, memory mem, immI_255 bytemask) %{
-  match(Set dst (AndI (LoadB mem) bytemask));
+// Load Unsigned Byte (8bit UNsigned) into an int reg
+instruct loadUB(iRegI dst, memory mem) %{
+  match(Set dst (LoadUB mem));
   ins_cost(MEMORY_REF_COST);
 
   size(4);
-  format %{ "LDUB   $mem,$dst" %}
+  format %{ "LDUB   $mem,$dst\t! ubyte" %}
+  opcode(Assembler::ldub_op3);
+  ins_encode(simple_form3_mem_reg( mem, dst ) );
+  ins_pipe(iload_mask_mem);
+%}
+
+// Load Unsigned Byte (8bit UNsigned) into a Long Register
+instruct loadUB2L(iRegL dst, memory mem) %{
+  match(Set dst (ConvI2L (LoadUB mem)));
+  ins_cost(MEMORY_REF_COST);
+
+  size(4);
+  format %{ "LDUB   $mem,$dst\t! ubyte -> long" %}
   opcode(Assembler::ldub_op3);
   ins_encode(simple_form3_mem_reg( mem, dst ) );
   ins_pipe(iload_mask_mem);
 %}
 
-// Load Byte (8bit UNsigned) into a Long Register
-instruct loadUBL(iRegL dst, memory mem, immL_FF bytemask) %{
-  match(Set dst (AndL (ConvI2L (LoadB mem)) bytemask));
+// Load Short (16bit signed)
+instruct loadS(iRegI dst, memory mem) %{
+  match(Set dst (LoadS mem));
+  ins_cost(MEMORY_REF_COST);
+
+  size(4);
+  format %{ "LDSH   $mem,$dst\t! short" %}
+  opcode(Assembler::ldsh_op3);
+  ins_encode(simple_form3_mem_reg( mem, dst ) );
+  ins_pipe(iload_mask_mem);
+%}
+
+// Load Short (16bit signed) into a Long Register
+instruct loadS2L(iRegL dst, memory mem) %{
+  match(Set dst (ConvI2L (LoadS mem)));
   ins_cost(MEMORY_REF_COST);
 
   size(4);
-  format %{ "LDUB   $mem,$dst" %}
-  opcode(Assembler::ldub_op3);
+  format %{ "LDSH   $mem,$dst\t! short -> long" %}
+  opcode(Assembler::ldsh_op3);
+  ins_encode(simple_form3_mem_reg( mem, dst ) );
+  ins_pipe(iload_mask_mem);
+%}
+
+// Load Unsigned Short/Char (16bit UNsigned)
+instruct loadUS(iRegI dst, memory mem) %{
+  match(Set dst (LoadUS mem));
+  ins_cost(MEMORY_REF_COST);
+
+  size(4);
+  format %{ "LDUH   $mem,$dst\t! ushort/char" %}
+  opcode(Assembler::lduh_op3);
   ins_encode(simple_form3_mem_reg( mem, dst ) );
   ins_pipe(iload_mask_mem);
 %}
 
 // Load Unsigned Short/Char (16bit UNsigned) into a Long Register
-instruct loadUS2L(iRegL dst, memory mem, immL_FFFF bytemask) %{
-  match(Set dst (AndL (ConvI2L (LoadUS mem)) bytemask));
+instruct loadUS2L(iRegL dst, memory mem) %{
+  match(Set dst (ConvI2L (LoadUS mem)));
   ins_cost(MEMORY_REF_COST);
 
   size(4);
-  format %{ "LDUH   $mem,$dst" %}
-  opcode(Assembler::lduh_op3);
-  ins_encode(simple_form3_mem_reg( mem, dst ) );
-  ins_pipe(iload_mask_mem);
-%}
-
-// Load Unsigned Short/Char (16bit unsigned)
-instruct loadUS(iRegI dst, memory mem) %{
-  match(Set dst (LoadUS mem));
-  ins_cost(MEMORY_REF_COST);
-
-  size(4);
-  format %{ "LDUH   $mem,$dst" %}
+  format %{ "LDUH   $mem,$dst\t! ushort/char -> long" %}
   opcode(Assembler::lduh_op3);
   ins_encode(simple_form3_mem_reg( mem, dst ) );
   ins_pipe(iload_mask_mem);
@@ -5344,9 +5380,33 @@
 instruct loadI(iRegI dst, memory mem) %{
   match(Set dst (LoadI mem));
   ins_cost(MEMORY_REF_COST);
-  size(4);
-
-  format %{ "LDUW   $mem,$dst" %}
+
+  size(4);
+  format %{ "LDUW   $mem,$dst\t! int" %}
+  opcode(Assembler::lduw_op3);
+  ins_encode(simple_form3_mem_reg( mem, dst ) );
+  ins_pipe(iload_mem);
+%}
+
+// Load Integer into a Long Register
+instruct loadI2L(iRegL dst, memory mem) %{
+  match(Set dst (ConvI2L (LoadI mem)));
+  ins_cost(MEMORY_REF_COST);
+
+  size(4);
+  format %{ "LDSW   $mem,$dst\t! int -> long" %}
+  opcode(Assembler::ldsw_op3);
+  ins_encode(simple_form3_mem_reg( mem, dst ) );
+  ins_pipe(iload_mem);
+%}
+
+// Load Unsigned Integer into a Long Register
+instruct loadUI2L(iRegL dst, memory mem) %{
+  match(Set dst (LoadUI2L mem));
+  ins_cost(MEMORY_REF_COST);
+
+  size(4);
+  format %{ "LDUW   $mem,$dst\t! uint -> long" %}
   opcode(Assembler::lduw_op3);
   ins_encode(simple_form3_mem_reg( mem, dst ) );
   ins_pipe(iload_mem);
@@ -5356,6 +5416,7 @@
 instruct loadL(iRegL dst, memory mem ) %{
   match(Set dst (LoadL mem));
   ins_cost(MEMORY_REF_COST);
+
   size(4);
   format %{ "LDX    $mem,$dst\t! long" %}
   opcode(Assembler::ldx_op3);
@@ -5471,13 +5532,11 @@
 
    format %{ "LDUW   $mem,$dst\t! compressed ptr" %}
    ins_encode %{
-     Register base = as_Register($mem$$base);
-     Register index = as_Register($mem$$index);
-     Register dst = $dst$$Register;
+     Register index = $mem$$index$$Register;
      if (index != G0) {
-       __ lduw(base, index, dst);
+       __ lduw($mem$$base$$Register, index, $dst$$Register);
      } else {
-       __ lduw(base, $mem$$disp, dst);
+       __ lduw($mem$$base$$Register, $mem$$disp, $dst$$Register);
      }
    %}
    ins_pipe(iload_mem);
@@ -5521,18 +5580,6 @@
   ins_pipe(iload_mem);
 %}
 
-// Load Short (16bit signed)
-instruct loadS(iRegI dst, memory mem) %{
-  match(Set dst (LoadS mem));
-  ins_cost(MEMORY_REF_COST);
-
-  size(4);
-  format %{ "LDSH   $mem,$dst" %}
-  opcode(Assembler::ldsh_op3);
-  ins_encode(simple_form3_mem_reg( mem, dst ) );
-  ins_pipe(iload_mask_mem);
-%}
-
 // Load Double
 instruct loadD(regD dst, memory mem) %{
   match(Set dst (LoadD mem));
--- a/src/cpu/sparc/vm/vtableStubs_sparc.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/sparc/vm/vtableStubs_sparc.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -106,6 +106,15 @@
   __ delayed()->nop();
 
   masm->flush();
+
+  if (PrintMiscellaneous && (WizardMode || Verbose)) {
+    tty->print_cr("vtable #%d at "PTR_FORMAT"[%d] left over: %d",
+                  vtable_index, s->entry_point(),
+                  (int)(s->code_end() - s->entry_point()),
+                  (int)(s->code_end() - __ pc()));
+  }
+  guarantee(__ pc() <= s->code_end(), "overflowed buffer");
+
   s->set_exception_points(npe_addr, ame_addr);
   return s;
 }
@@ -113,9 +122,9 @@
 
 // NOTE:  %%%% if any change is made to this stub make sure that the function
 //             pd_code_size_limit is changed to ensure the correct size for VtableStub
-VtableStub* VtableStubs::create_itable_stub(int vtable_index) {
+VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   const int sparc_code_length = VtableStub::pd_code_size_limit(false);
-  VtableStub* s = new(sparc_code_length) VtableStub(false, vtable_index);
+  VtableStub* s = new(sparc_code_length) VtableStub(false, itable_index);
   ResourceMark rm;
   CodeBuffer cb(s->entry_point(), sparc_code_length);
   MacroAssembler* masm = new MacroAssembler(&cb);
@@ -139,7 +148,6 @@
   // are passed in the %o registers.  Instead, longs are passed in G1 and G4
   // and so those registers are not available here.
   __ save(SP,-frame::register_save_words*wordSize,SP);
-  Register I0_receiver = I0;    // Location of receiver after save
 
 #ifndef PRODUCT
   if (CountCompiledCalls) {
@@ -151,63 +159,31 @@
   }
 #endif /* PRODUCT */
 
-  // load start of itable entries into L0 register
-  const int base = instanceKlass::vtable_start_offset() * wordSize;
-  __ ld(Address(G3_klassOop, 0, instanceKlass::vtable_length_offset() * wordSize), L0);
-
-  // %%% Could store the aligned, prescaled offset in the klassoop.
-  __ sll(L0, exact_log2(vtableEntry::size() * wordSize), L0);
-  // see code for instanceKlass::start_of_itable!
-  const int vtable_alignment = align_object_offset(1);
-  assert(vtable_alignment == 1 || vtable_alignment == 2, "");
-  const int odd_bit = vtableEntry::size() * wordSize;
-  if (vtable_alignment == 2) {
-    __ and3(L0, odd_bit, L1);   // isolate the odd bit
-  }
-  __ add(G3_klassOop, L0, L0);
-  if (vtable_alignment == 2) {
-    __ add(L0, L1, L0);         // double the odd bit, to align up
-  }
+  Label throw_icce;
 
-  // Loop over all itable entries until desired interfaceOop (G5_interface) found
-  __ bind(search);
-
-  // %%%% Could load both offset and interface in one ldx, if they were
-  // in the opposite order.  This would save a load.
-  __ ld_ptr(L0, base + itableOffsetEntry::interface_offset_in_bytes(), L1);
-
-  // If the entry is NULL then we've reached the end of the table
-  // without finding the expected interface, so throw an exception
-  Label throw_icce;
-  __ bpr(Assembler::rc_z, false, Assembler::pn, L1, throw_icce);
-  __ delayed()->cmp(G5_interface, L1);
-  __ brx(Assembler::notEqual, true, Assembler::pn, search);
-  __ delayed()->add(L0, itableOffsetEntry::size() * wordSize, L0);
-
-  // entry found and L0 points to it, move offset of vtable for interface into L0
-  __ ld(L0, base + itableOffsetEntry::offset_offset_in_bytes(), L0);
-
-  // Compute itableMethodEntry and get methodOop(G5_method) and entrypoint(L0) for compiler
-  const int method_offset = (itableMethodEntry::size() * wordSize * vtable_index) + itableMethodEntry::method_offset_in_bytes();
-  __ add(G3_klassOop, L0, L1);
-  __ ld_ptr(L1, method_offset, G5_method);
+  Register L5_method = L5;
+  __ lookup_interface_method(// inputs: rec. class, interface, itable index
+                             G3_klassOop, G5_interface, itable_index,
+                             // outputs: method, scan temp. reg
+                             L5_method, L2, L3,
+                             throw_icce);
 
 #ifndef PRODUCT
   if (DebugVtables) {
     Label L01;
-    __ ld_ptr(L1, method_offset, G5_method);
-    __ bpr(Assembler::rc_nz, false, Assembler::pt, G5_method, L01);
+    __ bpr(Assembler::rc_nz, false, Assembler::pt, L5_method, L01);
     __ delayed()->nop();
     __ stop("methodOop is null");
     __ bind(L01);
-    __ verify_oop(G5_method);
+    __ verify_oop(L5_method);
   }
 #endif
 
   // If the following load is through a NULL pointer, we'll take an OS
   // exception that should translate into an AbstractMethodError.  We need the
   // window count to be correct at that time.
-  __ restore();                 // Restore registers BEFORE the AME point
+  __ restore(L5_method, 0, G5_method);
+  // Restore registers *before* the AME point.
 
   address ame_addr = __ pc();   // if the vtable entry is null, the method is abstract
   __ ld_ptr(G5_method, in_bytes(methodOopDesc::from_compiled_offset()), G3_scratch);
@@ -225,6 +201,12 @@
 
   masm->flush();
 
+  if (PrintMiscellaneous && (WizardMode || Verbose)) {
+    tty->print_cr("itable #%d at "PTR_FORMAT"[%d] left over: %d",
+                  itable_index, s->entry_point(),
+                  (int)(s->code_end() - s->entry_point()),
+                  (int)(s->code_end() - __ pc()));
+  }
   guarantee(__ pc() <= s->code_end(), "overflowed buffer");
 
   s->set_exception_points(npe_addr, ame_addr);
@@ -243,8 +225,7 @@
                         (UseCompressedOops ? 2*BytesPerInstWord : 0);
       return basic + slop;
     } else {
-      // save, ld, ld, sll, and, add, add, ld, cmp, br, add, ld, add, ld, ld, jmp, restore, sethi, jmpl, restore
-      const int basic = (20 LP64_ONLY(+ 6)) * BytesPerInstWord +
+      const int basic = (28 LP64_ONLY(+ 6)) * BytesPerInstWord +
                         // shift;add for load_klass
                         (UseCompressedOops ? 2*BytesPerInstWord : 0);
       return (basic + slop);
--- a/src/cpu/x86/vm/assembler_x86.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/assembler_x86.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -129,13 +129,19 @@
 // Convert the raw encoding form into the form expected by the constructor for
 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 // that to noreg for the Address constructor.
-Address Address::make_raw(int base, int index, int scale, int disp) {
+Address Address::make_raw(int base, int index, int scale, int disp, bool disp_is_oop) {
+  RelocationHolder rspec;
+  if (disp_is_oop) {
+    rspec = Relocation::spec_simple(relocInfo::oop_type);
+  }
   bool valid_index = index != rsp->encoding();
   if (valid_index) {
     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
+    madr._rspec = rspec;
     return madr;
   } else {
     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
+    madr._rspec = rspec;
     return madr;
   }
 }
@@ -3892,6 +3898,21 @@
   emit_operand(src, dst);
 }
 
+void Assembler::movsbq(Register dst, Address src) {
+  InstructionMark im(this);
+  prefixq(src, dst);
+  emit_byte(0x0F);
+  emit_byte(0xBE);
+  emit_operand(dst, src);
+}
+
+void Assembler::movsbq(Register dst, Register src) {
+  int encode = prefixq_and_encode(dst->encoding(), src->encoding());
+  emit_byte(0x0F);
+  emit_byte(0xBE);
+  emit_byte(0xC0 | encode);
+}
+
 void Assembler::movslq(Register dst, int32_t imm32) {
   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
@@ -3925,6 +3946,51 @@
   emit_byte(0xC0 | encode);
 }
 
+void Assembler::movswq(Register dst, Address src) {
+  InstructionMark im(this);
+  prefixq(src, dst);
+  emit_byte(0x0F);
+  emit_byte(0xBF);
+  emit_operand(dst, src);
+}
+
+void Assembler::movswq(Register dst, Register src) {
+  int encode = prefixq_and_encode(dst->encoding(), src->encoding());
+  emit_byte(0x0F);
+  emit_byte(0xBF);
+  emit_byte(0xC0 | encode);
+}
+
+void Assembler::movzbq(Register dst, Address src) {
+  InstructionMark im(this);
+  prefixq(src, dst);
+  emit_byte(0x0F);
+  emit_byte(0xB6);
+  emit_operand(dst, src);
+}
+
+void Assembler::movzbq(Register dst, Register src) {
+  int encode = prefixq_and_encode(dst->encoding(), src->encoding());
+  emit_byte(0x0F);
+  emit_byte(0xB6);
+  emit_byte(0xC0 | encode);
+}
+
+void Assembler::movzwq(Register dst, Address src) {
+  InstructionMark im(this);
+  prefixq(src, dst);
+  emit_byte(0x0F);
+  emit_byte(0xB7);
+  emit_operand(dst, src);
+}
+
+void Assembler::movzwq(Register dst, Register src) {
+  int encode = prefixq_and_encode(dst->encoding(), src->encoding());
+  emit_byte(0x0F);
+  emit_byte(0xB7);
+  emit_byte(0xC0 | encode);
+}
+
 void Assembler::negq(Register dst) {
   int encode = prefixq_and_encode(dst->encoding());
   emit_byte(0xF7);
@@ -6197,8 +6263,11 @@
   return off;
 }
 
-// word => int32 which seems bad for 64bit
-int MacroAssembler::load_signed_word(Register dst, Address src) {
+// Note: load_signed_short used to be called load_signed_word.
+// Although the 'w' in x86 opcodes refers to the term "word" in the assembler
+// manual, which means 16 bits, that usage is found nowhere in HotSpot code.
+// The term "word" in HotSpot means a 32- or 64-bit machine word.
+int MacroAssembler::load_signed_short(Register dst, Address src) {
   int off;
   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
@@ -6207,7 +6276,7 @@
     off = offset();
     movswl(dst, src); // movsxw
   } else {
-    off = load_unsigned_word(dst, src);
+    off = load_unsigned_short(dst, src);
     shll(dst, 16);
     sarl(dst, 16);
   }
@@ -6229,7 +6298,8 @@
   return off;
 }
 
-int MacroAssembler::load_unsigned_word(Register dst, Address src) {
+// Note: load_unsigned_short used to be called load_unsigned_word.
+int MacroAssembler::load_unsigned_short(Register dst, Address src) {
   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
   // and "3.9 Partial Register Penalties", p. 22).
   int off;
@@ -6244,6 +6314,28 @@
   return off;
 }
 
+void MacroAssembler::load_sized_value(Register dst, Address src,
+                                      int size_in_bytes, bool is_signed) {
+  switch (size_in_bytes ^ (is_signed ? -1 : 0)) {
+#ifndef _LP64
+  // For case 8, caller is responsible for manually loading
+  // the second word into another register.
+  case ~8:  // fall through:
+  case  8:  movl(                dst, src ); break;
+#else
+  case ~8:  // fall through:
+  case  8:  movq(                dst, src ); break;
+#endif
+  case ~4:  // fall through:
+  case  4:  movl(                dst, src ); break;
+  case ~2:  load_signed_short(   dst, src ); break;
+  case  2:  load_unsigned_short( dst, src ); break;
+  case ~1:  load_signed_byte(    dst, src ); break;
+  case  1:  load_unsigned_byte(  dst, src ); break;
+  default:  ShouldNotReachHere();
+  }
+}
+
 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
   if (reachable(dst)) {
     movl(as_Address(dst), src);
@@ -7050,6 +7142,81 @@
 }
 
 
+// Look up the method for a megamorphic invokeinterface call.
+// The target method is determined by <intf_klass, itable_index>.
+// The receiver klass is in recv_klass.
+// On success, the result will be in method_result, and execution falls through.
+// On failure, execution transfers to the given label.
+void MacroAssembler::lookup_interface_method(Register recv_klass,
+                                             Register intf_klass,
+                                             RegisterConstant itable_index,
+                                             Register method_result,
+                                             Register scan_temp,
+                                             Label& L_no_such_interface) {
+  assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
+  assert(itable_index.is_constant() || itable_index.as_register() == method_result,
+         "caller must use same register for non-constant itable index as for method");
+
+  // Compute start of first itableOffsetEntry (which is at the end of the vtable)
+  int vtable_base = instanceKlass::vtable_start_offset() * wordSize;
+  int itentry_off = itableMethodEntry::method_offset_in_bytes();
+  int scan_step   = itableOffsetEntry::size() * wordSize;
+  int vte_size    = vtableEntry::size() * wordSize;
+  Address::ScaleFactor times_vte_scale = Address::times_ptr;
+  assert(vte_size == wordSize, "else adjust times_vte_scale");
+
+  movl(scan_temp, Address(recv_klass, instanceKlass::vtable_length_offset() * wordSize));
+
+  // %%% Could store the aligned, prescaled offset in the klassoop.
+  lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
+  if (HeapWordsPerLong > 1) {
+    // Round up to align_object_offset boundary
+    // see code for instanceKlass::start_of_itable!
+    round_to(scan_temp, BytesPerLong);
+  }
+
+  // Adjust recv_klass by scaled itable_index, so we can free itable_index.
+  assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
+  lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
+
+  // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
+  //   if (scan->interface() == intf) {
+  //     result = (klass + scan->offset() + itable_index);
+  //   }
+  // }
+  Label search, found_method;
+
+  for (int peel = 1; peel >= 0; peel--) {
+    movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
+    cmpptr(intf_klass, method_result);
+
+    if (peel) {
+      jccb(Assembler::equal, found_method);
+    } else {
+      jccb(Assembler::notEqual, search);
+      // (invert the test to fall through to found_method...)
+    }
+
+    if (!peel)  break;
+
+    bind(search);
+
+    // Check that the previous entry is non-null.  A null entry means that
+    // the receiver class doesn't implement the interface, and wasn't the
+    // same as when the caller was compiled.
+    testptr(method_result, method_result);
+    jcc(Assembler::zero, L_no_such_interface);
+    addptr(scan_temp, scan_step);
+  }
+
+  bind(found_method);
+
+  // Got a hit.
+  movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
+  movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
+}
+
+
 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
   ucomisd(dst, as_Address(src));
 }
@@ -7095,6 +7262,31 @@
 }
 
 
+RegisterConstant MacroAssembler::delayed_value(intptr_t* delayed_value_addr,
+                                               Register tmp,
+                                               int offset) {
+  intptr_t value = *delayed_value_addr;
+  if (value != 0)
+    return RegisterConstant(value + offset);
+
+  // load indirectly to solve generation ordering problem
+  movptr(tmp, ExternalAddress((address) delayed_value_addr));
+
+#ifdef ASSERT
+  Label L;
+  testl(tmp, tmp);
+  jccb(Assembler::notZero, L);
+  hlt();
+  bind(L);
+#endif
+
+  if (offset != 0)
+    addptr(tmp, offset);
+
+  return RegisterConstant(tmp);
+}
+
+
 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
   if (!VerifyOops) return;
 
--- a/src/cpu/x86/vm/assembler_x86.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/assembler_x86.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -153,6 +153,21 @@
     times_8  =  3,
     times_ptr = LP64_ONLY(times_8) NOT_LP64(times_4)
   };
+  static ScaleFactor times(int size) {
+    assert(size >= 1 && size <= 8 && is_power_of_2(size), "bad scale size");
+    if (size == 8)  return times_8;
+    if (size == 4)  return times_4;
+    if (size == 2)  return times_2;
+    return times_1;
+  }
+  static int scale_size(ScaleFactor scale) {
+    assert(scale != no_scale, "");
+    assert(((1 << (int)times_1) == 1 &&
+            (1 << (int)times_2) == 2 &&
+            (1 << (int)times_4) == 4 &&
+            (1 << (int)times_8) == 8), "");
+    return (1 << (int)scale);
+  }
 
  private:
   Register         _base;
@@ -197,6 +212,22 @@
            "inconsistent address");
   }
 
+  Address(Register base, RegisterConstant index, ScaleFactor scale = times_1, int disp = 0)
+    : _base (base),
+      _index(index.register_or_noreg()),
+      _scale(scale),
+      _disp (disp + (index.constant_or_zero() * scale_size(scale))) {
+    if (!index.is_register())  scale = Address::no_scale;
+    assert(!_index->is_valid() == (scale == Address::no_scale),
+           "inconsistent address");
+  }
+
+  Address plus_disp(int disp) const {
+    Address a = (*this);
+    a._disp += disp;
+    return a;
+  }
+
   // The following two overloads are used in connection with the
   // ByteSize type (see sizes.hpp).  They simplify the use of
   // ByteSize'd arguments in assembly code. Note that their equivalent
@@ -224,6 +255,17 @@
     assert(!index->is_valid() == (scale == Address::no_scale),
            "inconsistent address");
   }
+
+  Address(Register base, RegisterConstant index, ScaleFactor scale, ByteSize disp)
+    : _base (base),
+      _index(index.register_or_noreg()),
+      _scale(scale),
+      _disp (in_bytes(disp) + (index.constant_or_zero() * scale_size(scale))) {
+    if (!index.is_register())  scale = Address::no_scale;
+    assert(!_index->is_valid() == (scale == Address::no_scale),
+           "inconsistent address");
+  }
+
 #endif // ASSERT
 
   // accessors
@@ -236,11 +278,10 @@
   // Convert the raw encoding form into the form expected by the constructor for
   // Address.  An index of 4 (rsp) corresponds to having no index, so convert
   // that to noreg for the Address constructor.
-  static Address make_raw(int base, int index, int scale, int disp);
+  static Address make_raw(int base, int index, int scale, int disp, bool disp_is_oop);
 
   static Address make_array(ArrayAddress);
 
-
  private:
   bool base_needs_rex() const {
     return _base != noreg && _base->encoding() >= 8;
@@ -1097,6 +1138,9 @@
   void movsbl(Register dst, Register src);
 
 #ifdef _LP64
+  void movsbq(Register dst, Address src);
+  void movsbq(Register dst, Register src);
+
   // Move signed 32bit immediate to 64bit extending sign
   void movslq(Address dst, int32_t imm64);
   void movslq(Register dst, int32_t imm64);
@@ -1109,6 +1153,11 @@
   void movswl(Register dst, Address src);
   void movswl(Register dst, Register src);
 
+#ifdef _LP64
+  void movswq(Register dst, Address src);
+  void movswq(Register dst, Register src);
+#endif
+
   void movw(Address dst, int imm16);
   void movw(Register dst, Address src);
   void movw(Address dst, Register src);
@@ -1116,9 +1165,19 @@
   void movzbl(Register dst, Address src);
   void movzbl(Register dst, Register src);
 
+#ifdef _LP64
+  void movzbq(Register dst, Address src);
+  void movzbq(Register dst, Register src);
+#endif
+
   void movzwl(Register dst, Address src);
   void movzwl(Register dst, Register src);
 
+#ifdef _LP64
+  void movzwq(Register dst, Address src);
+  void movzwq(Register dst, Register src);
+#endif
+
   void mull(Address src);
   void mull(Register src);
 
@@ -1393,17 +1452,20 @@
 
   // The following 4 methods return the offset of the appropriate move instruction
 
-  // Support for fast byte/word loading with zero extension (depending on particular CPU)
+  // Support for fast byte/short loading with zero extension (depending on particular CPU)
   int load_unsigned_byte(Register dst, Address src);
-  int load_unsigned_word(Register dst, Address src);
-
-  // Support for fast byte/word loading with sign extension (depending on particular CPU)
+  int load_unsigned_short(Register dst, Address src);
+
+  // Support for fast byte/short loading with sign extension (depending on particular CPU)
   int load_signed_byte(Register dst, Address src);
-  int load_signed_word(Register dst, Address src);
+  int load_signed_short(Register dst, Address src);
 
   // Support for sign-extension (hi:lo = extend_sign(lo))
   void extend_sign(Register hi, Register lo);
 
+  // Loading values by size and signed-ness
+  void load_sized_value(Register dst, Address src, int size_in_bytes, bool is_signed);
+
   // Support for inc/dec with optimal instruction selection depending on value
 
   void increment(Register reg, int value = 1) { LP64_ONLY(incrementq(reg, value)) NOT_LP64(incrementl(reg, value)) ; }
@@ -1721,6 +1783,14 @@
   );
   void tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case);
 
+  // interface method calling
+  void lookup_interface_method(Register recv_klass,
+                               Register intf_klass,
+                               RegisterConstant itable_index,
+                               Register method_result,
+                               Register scan_temp,
+                               Label& no_such_interface);
+
   //----
   void set_word_if_not_zero(Register reg); // sets reg to 1 if not zero, otherwise 0
 
@@ -1763,6 +1833,10 @@
   // stack overflow + shadow pages.  Also, clobbers tmp
   void bang_stack_size(Register size, Register tmp);
 
+  virtual RegisterConstant delayed_value(intptr_t* delayed_value_addr,
+                                         Register tmp,
+                                         int offset);
+
   // Support for serializing memory accesses between threads
   void serialize_memory(Register thread, Register tmp);
 
--- a/src/cpu/x86/vm/bytecodeInterpreter_x86.inline.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/bytecodeInterpreter_x86.inline.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -554,8 +554,8 @@
   __ jcc (Assembler::zero, noLoop);
 
   // compare first characters
-  __ load_unsigned_word(rcx, Address(rdi, 0));
-  __ load_unsigned_word(rbx, Address(rsi, 0));
+  __ load_unsigned_short(rcx, Address(rdi, 0));
+  __ load_unsigned_short(rbx, Address(rsi, 0));
   __ subl(rcx, rbx);
   __ jcc(Assembler::notZero, haveResult);
   // starting loop
@@ -574,8 +574,8 @@
   Label loop;
   __ align(wordSize);
   __ bind(loop);
-  __ load_unsigned_word(rcx, Address(rdi, rax, Address::times_2, 0));
-  __ load_unsigned_word(rbx, Address(rsi, rax, Address::times_2, 0));
+  __ load_unsigned_short(rcx, Address(rdi, rax, Address::times_2, 0));
+  __ load_unsigned_short(rbx, Address(rsi, rax, Address::times_2, 0));
   __ subl(rcx, rbx);
   __ jcc(Assembler::notZero, haveResult);
   __ increment(rax);
--- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/cppInterpreter_x86.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/cppInterpreter_x86.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2007-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2007-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -513,7 +513,7 @@
     // compute full expression stack limit
 
     const Address size_of_stack    (rbx, methodOopDesc::max_stack_offset());
-    __ load_unsigned_word(rdx, size_of_stack);                            // get size of expression stack in words
+    __ load_unsigned_short(rdx, size_of_stack);                           // get size of expression stack in words
     __ negptr(rdx);                                                       // so we can subtract in next step
     // Allocate expression stack
     __ lea(rsp, Address(rsp, rdx, Address::times_ptr));
@@ -659,7 +659,7 @@
     // Always give one monitor to allow us to start interp if sync method.
     // Any additional monitors need a check when moving the expression stack
     const int one_monitor = frame::interpreter_frame_monitor_size() * wordSize;
-  __ load_unsigned_word(rax, size_of_stack);                            // get size of expression stack in words
+  __ load_unsigned_short(rax, size_of_stack);                           // get size of expression stack in words
   __ lea(rax, Address(noreg, rax, Interpreter::stackElementScale(), one_monitor));
   __ lea(rax, Address(rax, rdx, Interpreter::stackElementScale(), overhead_size));
 
@@ -863,13 +863,13 @@
     __ bind(notByte);
     __ cmpl(rdx, stos);
     __ jcc(Assembler::notEqual, notShort);
-    __ load_signed_word(rax, field_address);
+    __ load_signed_short(rax, field_address);
     __ jmp(xreturn_path);
 
     __ bind(notShort);
     __ cmpl(rdx, ctos);
     __ jcc(Assembler::notEqual, notChar);
-    __ load_unsigned_word(rax, field_address);
+    __ load_unsigned_short(rax, field_address);
     __ jmp(xreturn_path);
 
     __ bind(notChar);
@@ -937,7 +937,7 @@
   const Register locals = rdi;
 
   // get parameter size (always needed)
-  __ load_unsigned_word(rcx, size_of_parameters);
+  __ load_unsigned_short(rcx, size_of_parameters);
 
   // rbx: methodOop
   // rcx: size of parameters
@@ -1062,7 +1062,7 @@
   // allocate space for parameters
   __ movptr(method, STATE(_method));
   __ verify_oop(method);
-  __ load_unsigned_word(t, Address(method, methodOopDesc::size_of_parameters_offset()));
+  __ load_unsigned_short(t, Address(method, methodOopDesc::size_of_parameters_offset()));
   __ shll(t, 2);
 #ifdef _LP64
   __ subptr(rsp, t);
@@ -1659,11 +1659,11 @@
   // const Address monitor(rbp, frame::interpreter_frame_initial_sp_offset * wordSize - (int)sizeof(BasicObjectLock));
 
   // get parameter size (always needed)
-  __ load_unsigned_word(rcx, size_of_parameters);
+  __ load_unsigned_short(rcx, size_of_parameters);
 
   // rbx: methodOop
   // rcx: size of parameters
-  __ load_unsigned_word(rdx, size_of_locals);                      // get size of locals in words
+  __ load_unsigned_short(rdx, size_of_locals);                     // get size of locals in words
 
   __ subptr(rdx, rcx);                                             // rdx = no. of additional locals
 
@@ -1949,7 +1949,7 @@
   __ movptr(rbx, STATE(_result._to_call._callee));
 
   // callee left args on top of expression stack, remove them
-  __ load_unsigned_word(rcx, Address(rbx, methodOopDesc::size_of_parameters_offset()));
+  __ load_unsigned_short(rcx, Address(rbx, methodOopDesc::size_of_parameters_offset()));
   __ lea(rsp, Address(rsp, rcx, Address::times_ptr));
 
   __ movl(rcx, Address(rbx, methodOopDesc::result_index_offset()));
@@ -2119,7 +2119,7 @@
   // Make it look like call_stub calling conventions
 
   // Get (potential) receiver
-  __ load_unsigned_word(rcx, size_of_parameters);                     // get size of parameters in words
+  __ load_unsigned_short(rcx, size_of_parameters);                   // get size of parameters in words
 
   ExternalAddress recursive(CAST_FROM_FN_PTR(address, RecursiveInterpreterActivation));
   __ pushptr(recursive.addr());                                      // make it look good in the debugger
--- a/src/cpu/x86/vm/frame_x86.inline.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/frame_x86.inline.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/interp_masm_x86_32.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/interp_masm_x86_32.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -192,7 +192,7 @@
 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset) {
   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
   assert(cache != index, "must use different registers");
-  load_unsigned_word(index, Address(rsi, bcp_offset));
+  load_unsigned_short(index, Address(rsi, bcp_offset));
   movptr(cache, Address(rbp, frame::interpreter_frame_cache_offset * wordSize));
   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
   shlptr(index, 2); // convert from field index to ConstantPoolCacheEntry index
@@ -202,7 +202,7 @@
 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register tmp, int bcp_offset) {
   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
   assert(cache != tmp, "must use different register");
-  load_unsigned_word(tmp, Address(rsi, bcp_offset));
+  load_unsigned_short(tmp, Address(rsi, bcp_offset));
   assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
                                // convert from field index to ConstantPoolCacheEntry index
                                // and from word offset to byte offset
@@ -1031,7 +1031,7 @@
 
   // If the mdp is valid, it will point to a DataLayout header which is
   // consistent with the bcp.  The converse is highly probable also.
-  load_unsigned_word(rdx, Address(rcx, in_bytes(DataLayout::bci_offset())));
+  load_unsigned_short(rdx, Address(rcx, in_bytes(DataLayout::bci_offset())));
   addptr(rdx, Address(rbx, methodOopDesc::const_offset()));
   lea(rdx, Address(rdx, constMethodOopDesc::codes_offset()));
   cmpptr(rdx, rsi);
--- a/src/cpu/x86/vm/interp_masm_x86_32.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/interp_masm_x86_32.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/interp_masm_x86_64.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/interp_masm_x86_64.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -190,7 +190,7 @@
                                                            int bcp_offset) {
   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
   assert(cache != index, "must use different registers");
-  load_unsigned_word(index, Address(r13, bcp_offset));
+  load_unsigned_short(index, Address(r13, bcp_offset));
   movptr(cache, Address(rbp, frame::interpreter_frame_cache_offset * wordSize));
   assert(sizeof(ConstantPoolCacheEntry) == 4 * wordSize, "adjust code below");
   // convert from field index to ConstantPoolCacheEntry index
@@ -203,7 +203,7 @@
                                                                int bcp_offset) {
   assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
   assert(cache != tmp, "must use different register");
-  load_unsigned_word(tmp, Address(r13, bcp_offset));
+  load_unsigned_short(tmp, Address(r13, bcp_offset));
   assert(sizeof(ConstantPoolCacheEntry) == 4 * wordSize, "adjust code below");
   // convert from field index to ConstantPoolCacheEntry index
   // and from word offset to byte offset
@@ -1063,8 +1063,8 @@
 
   // If the mdp is valid, it will point to a DataLayout header which is
   // consistent with the bcp.  The converse is highly probable also.
-  load_unsigned_word(c_rarg2,
-                     Address(c_rarg3, in_bytes(DataLayout::bci_offset())));
+  load_unsigned_short(c_rarg2,
+                      Address(c_rarg3, in_bytes(DataLayout::bci_offset())));
   addptr(c_rarg2, Address(rbx, methodOopDesc::const_offset()));
   lea(c_rarg2, Address(c_rarg2, constMethodOopDesc::codes_offset()));
   cmpptr(c_rarg2, r13);
--- a/src/cpu/x86/vm/interpreterRT_x86_32.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/interpreterRT_x86_32.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/jni_x86.h	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/jni_x86.h	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/runtime_x86_32.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/runtime_x86_32.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/sharedRuntime_x86_32.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/sharedRuntime_x86_32.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/sharedRuntime_x86_64.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/stubGenerator_x86_32.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/stubGenerator_x86_32.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/cpu/x86/vm/templateInterpreter_x86_32.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/templateInterpreter_x86_32.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -662,13 +662,13 @@
     __ bind(notByte);
     __ cmpl(rdx, stos);
     __ jcc(Assembler::notEqual, notShort);
-    __ load_signed_word(rax, field_address);
+    __ load_signed_short(rax, field_address);
     __ jmp(xreturn_path);
 
     __ bind(notShort);
     __ cmpl(rdx, ctos);
     __ jcc(Assembler::notEqual, notChar);
-    __ load_unsigned_word(rax, field_address);
+    __ load_unsigned_short(rax, field_address);
     __ jmp(xreturn_path);
 
     __ bind(notChar);
@@ -723,7 +723,7 @@
   const Address access_flags      (rbx, methodOopDesc::access_flags_offset());
 
   // get parameter size (always needed)
-  __ load_unsigned_word(rcx, size_of_parameters);
+  __ load_unsigned_short(rcx, size_of_parameters);
 
   // native calls don't need the stack size check since they have no expression stack
   // and the arguments are already on the stack and we only add a handful of words
@@ -838,7 +838,7 @@
   // allocate space for parameters
   __ get_method(method);
   __ verify_oop(method);
-  __ load_unsigned_word(t, Address(method, methodOopDesc::size_of_parameters_offset()));
+  __ load_unsigned_short(t, Address(method, methodOopDesc::size_of_parameters_offset()));
   __ shlptr(t, Interpreter::logStackElementSize());
   __ addptr(t, 2*wordSize);     // allocate two more slots for JNIEnv and possible mirror
   __ subptr(rsp, t);
@@ -1155,14 +1155,14 @@
   const Address access_flags      (rbx, methodOopDesc::access_flags_offset());
 
   // get parameter size (always needed)
-  __ load_unsigned_word(rcx, size_of_parameters);
+  __ load_unsigned_short(rcx, size_of_parameters);
 
   // rbx,: methodOop
   // rcx: size of parameters
 
   // rsi: sender_sp (could differ from sp+wordSize if we were called via c2i )
 
-  __ load_unsigned_word(rdx, size_of_locals);       // get size of locals in words
+  __ load_unsigned_short(rdx, size_of_locals);       // get size of locals in words
   __ subl(rdx, rcx);                                // rdx = no. of additional locals
 
   // see if we've got enough room on the stack for locals plus overhead.
@@ -1558,7 +1558,7 @@
     // Compute size of arguments for saving when returning to deoptimized caller
     __ get_method(rax);
     __ verify_oop(rax);
-    __ load_unsigned_word(rax, Address(rax, in_bytes(methodOopDesc::size_of_parameters_offset())));
+    __ load_unsigned_short(rax, Address(rax, in_bytes(methodOopDesc::size_of_parameters_offset())));
     __ shlptr(rax, Interpreter::logStackElementSize());
     __ restore_locals();
     __ subptr(rdi, rax);
--- a/src/cpu/x86/vm/templateInterpreter_x86_64.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/templateInterpreter_x86_64.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -650,7 +650,7 @@
     __ cmpl(rdx, stos);
     __ jcc(Assembler::notEqual, notShort);
     // stos
-    __ load_signed_word(rax, field_address);
+    __ load_signed_short(rax, field_address);
     __ jmp(xreturn_path);
 
     __ bind(notShort);
@@ -662,7 +662,7 @@
     __ bind(okay);
 #endif
     // ctos
-    __ load_unsigned_word(rax, field_address);
+    __ load_unsigned_short(rax, field_address);
 
     __ bind(xreturn_path);
 
@@ -702,7 +702,7 @@
   const Address access_flags      (rbx, methodOopDesc::access_flags_offset());
 
   // get parameter size (always needed)
-  __ load_unsigned_word(rcx, size_of_parameters);
+  __ load_unsigned_short(rcx, size_of_parameters);
 
   // native calls don't need the stack size check since they have no
   // expression stack and the arguments are already on the stack and
@@ -819,9 +819,9 @@
   // allocate space for parameters
   __ get_method(method);
   __ verify_oop(method);
-  __ load_unsigned_word(t,
-                        Address(method,
-                                methodOopDesc::size_of_parameters_offset()));
+  __ load_unsigned_short(t,
+                         Address(method,
+                                 methodOopDesc::size_of_parameters_offset()));
   __ shll(t, Interpreter::logStackElementSize());
 
   __ subptr(rsp, t);
@@ -1165,13 +1165,13 @@
   const Address access_flags(rbx, methodOopDesc::access_flags_offset());
 
   // get parameter size (always needed)
-  __ load_unsigned_word(rcx, size_of_parameters);
+  __ load_unsigned_short(rcx, size_of_parameters);
 
   // rbx: methodOop
   // rcx: size of parameters
   // r13: sender_sp (could differ from sp+wordSize if we were called via c2i )
 
-  __ load_unsigned_word(rdx, size_of_locals); // get size of locals in words
+  __ load_unsigned_short(rdx, size_of_locals); // get size of locals in words
   __ subl(rdx, rcx); // rdx = no. of additional locals
 
   // YYY
@@ -1583,7 +1583,7 @@
     // Compute size of arguments for saving when returning to
     // deoptimized caller
     __ get_method(rax);
-    __ load_unsigned_word(rax, Address(rax, in_bytes(methodOopDesc::
+    __ load_unsigned_short(rax, Address(rax, in_bytes(methodOopDesc::
                                                 size_of_parameters_offset())));
     __ shll(rax, Interpreter::logStackElementSize());
     __ restore_locals(); // XXX do we need this?
--- a/src/cpu/x86/vm/templateTable_x86_32.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/templateTable_x86_32.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -296,7 +296,7 @@
 
 void TemplateTable::sipush() {
   transition(vtos, itos);
-  __ load_unsigned_word(rax, at_bcp(1));
+  __ load_unsigned_short(rax, at_bcp(1));
   __ bswapl(rax);
   __ sarl(rax, 16);
 }
@@ -662,7 +662,7 @@
   index_check(rdx, rax);  // kills rbx,
   // rax,: index
   // can do better code for P5 - may want to improve this at some point
-  __ load_unsigned_word(rbx, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
+  __ load_unsigned_short(rbx, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
   __ mov(rax, rbx);
 }
 
@@ -677,7 +677,7 @@
   // rdx: array
   index_check(rdx, rax);
   // rax,: index
-  __ load_unsigned_word(rbx, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
+  __ load_unsigned_short(rbx, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
   __ mov(rax, rbx);
 }
 
@@ -687,7 +687,7 @@
   index_check(rdx, rax);  // kills rbx,
   // rax,: index
   // can do better code for P5 - may want to improve this at some point
-  __ load_signed_word(rbx, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_SHORT)));
+  __ load_signed_short(rbx, Address(rdx, rax, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_SHORT)));
   __ mov(rax, rbx);
 }
 
@@ -2310,7 +2310,7 @@
   __ cmpl(flags, ctos );
   __ jcc(Assembler::notEqual, notChar);
 
-  __ load_unsigned_word(rax, lo );
+  __ load_unsigned_short(rax, lo );
   __ push(ctos);
   if (!is_static) {
     patch_bytecode(Bytecodes::_fast_cgetfield, rcx, rbx);
@@ -2322,7 +2322,7 @@
   __ cmpl(flags, stos );
   __ jcc(Assembler::notEqual, notShort);
 
-  __ load_signed_word(rax, lo );
+  __ load_signed_short(rax, lo );
   __ push(stos);
   if (!is_static) {
     patch_bytecode(Bytecodes::_fast_sgetfield, rcx, rbx);
@@ -2830,8 +2830,8 @@
   // access field
   switch (bytecode()) {
     case Bytecodes::_fast_bgetfield: __ movsbl(rax, lo );                 break;
-    case Bytecodes::_fast_sgetfield: __ load_signed_word(rax, lo );       break;
-    case Bytecodes::_fast_cgetfield: __ load_unsigned_word(rax, lo );     break;
+    case Bytecodes::_fast_sgetfield: __ load_signed_short(rax, lo );      break;
+    case Bytecodes::_fast_cgetfield: __ load_unsigned_short(rax, lo );    break;
     case Bytecodes::_fast_igetfield: __ movl(rax, lo);                    break;
     case Bytecodes::_fast_lgetfield: __ stop("should not be rewritten");  break;
     case Bytecodes::_fast_fgetfield: __ fld_s(lo);                        break;
@@ -3055,35 +3055,44 @@
   // profile this call
   __ profile_virtual_call(rdx, rsi, rdi);
 
-  __ mov(rdi, rdx); // Save klassOop in rdi
-
-  // Compute start of first itableOffsetEntry (which is at the end of the vtable)
-  const int base = instanceKlass::vtable_start_offset() * wordSize;
-  assert(vtableEntry::size() * wordSize == (1 << (int)Address::times_ptr), "adjust the scaling in the code below");
-  __ movl(rsi, Address(rdx, instanceKlass::vtable_length_offset() * wordSize)); // Get length of vtable
-  __ lea(rdx, Address(rdx, rsi, Address::times_4, base));
-  if (HeapWordsPerLong > 1) {
-    // Round up to align_object_offset boundary
-    __ round_to(rdx, BytesPerLong);
-  }
-
-  Label entry, search, interface_ok;
-
-  __ jmpb(entry);
-  __ bind(search);
-  __ addptr(rdx, itableOffsetEntry::size() * wordSize);
-
-  __ bind(entry);
-
-  // Check that the entry is non-null.  A null entry means that the receiver
-  // class doesn't implement the interface, and wasn't the same as the
-  // receiver class checked when the interface was resolved.
-  __ push(rdx);
-  __ movptr(rdx, Address(rdx, itableOffsetEntry::interface_offset_in_bytes()));
-  __ testptr(rdx, rdx);
-  __ jcc(Assembler::notZero, interface_ok);
+  Label no_such_interface, no_such_method;
+
+  __ lookup_interface_method(// inputs: rec. class, interface, itable index
+                             rdx, rax, rbx,
+                             // outputs: method, scan temp. reg
+                             rbx, rsi,
+                             no_such_interface);
+
+  // rbx,: methodOop to call
+  // rcx: receiver
+  // Check for abstract method error
+  // Note: This should be done more efficiently via a throw_abstract_method_error
+  //       interpreter entry point and a conditional jump to it in case of a null
+  //       method.
+  __ testptr(rbx, rbx);
+  __ jcc(Assembler::zero, no_such_method);
+
+  // do the call
+  // rcx: receiver
+  // rbx,: methodOop
+  __ jump_from_interpreted(rbx, rdx);
+  __ should_not_reach_here();
+
+  // exception handling code follows...
+  // note: must restore interpreter registers to canonical
+  //       state for exception handling to work correctly!
+
+  __ bind(no_such_method);
   // throw exception
-  __ pop(rdx);           // pop saved register first.
+  __ pop(rbx);           // pop return address (pushed by prepare_invoke)
+  __ restore_bcp();      // rsi must be correct for exception handler   (was destroyed)
+  __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
+  __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
+  // the call_VM checks for exception, so we should never return here.
+  __ should_not_reach_here();
+
+  __ bind(no_such_interface);
+  // throw exception
   __ pop(rbx);           // pop return address (pushed by prepare_invoke)
   __ restore_bcp();      // rsi must be correct for exception handler   (was destroyed)
   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
@@ -3091,42 +3100,6 @@
                    InterpreterRuntime::throw_IncompatibleClassChangeError));
   // the call_VM checks for exception, so we should never return here.
   __ should_not_reach_here();
-  __ bind(interface_ok);
-
-    __ pop(rdx);
-
-    __ cmpptr(rax, Address(rdx, itableOffsetEntry::interface_offset_in_bytes()));
-    __ jcc(Assembler::notEqual, search);
-
-    __ movl(rdx, Address(rdx, itableOffsetEntry::offset_offset_in_bytes()));
-    __ addptr(rdx, rdi); // Add offset to klassOop
-    assert(itableMethodEntry::size() * wordSize == (1 << (int)Address::times_ptr), "adjust the scaling in the code below");
-    __ movptr(rbx, Address(rdx, rbx, Address::times_ptr));
-    // rbx,: methodOop to call
-    // rcx: receiver
-    // Check for abstract method error
-    // Note: This should be done more efficiently via a throw_abstract_method_error
-    //       interpreter entry point and a conditional jump to it in case of a null
-    //       method.
-    { Label L;
-      __ testptr(rbx, rbx);
-      __ jcc(Assembler::notZero, L);
-      // throw exception
-          // note: must restore interpreter registers to canonical
-          //       state for exception handling to work correctly!
-          __ pop(rbx);           // pop return address (pushed by prepare_invoke)
-          __ restore_bcp();      // rsi must be correct for exception handler   (was destroyed)
-          __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
-      __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
-      // the call_VM checks for exception, so we should never return here.
-      __ should_not_reach_here();
-      __ bind(L);
-    }
-
-  // do the call
-  // rcx: receiver
-  // rbx,: methodOop
-  __ jump_from_interpreted(rbx, rdx);
 }
 
 //----------------------------------------------------------------------------------------------------
--- a/src/cpu/x86/vm/templateTable_x86_64.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/templateTable_x86_64.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -307,7 +307,7 @@
 
 void TemplateTable::sipush() {
   transition(vtos, itos);
-  __ load_unsigned_word(rax, at_bcp(1));
+  __ load_unsigned_short(rax, at_bcp(1));
   __ bswapl(rax);
   __ sarl(rax, 16);
 }
@@ -645,10 +645,10 @@
   // eax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
-  __ load_unsigned_word(rax,
-                        Address(rdx, rax,
-                                Address::times_2,
-                                arrayOopDesc::base_offset_in_bytes(T_CHAR)));
+  __ load_unsigned_short(rax,
+                         Address(rdx, rax,
+                                 Address::times_2,
+                                 arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 }
 
 // iload followed by caload frequent pair
@@ -663,10 +663,10 @@
   // rdx: array
   __ pop_ptr(rdx);
   index_check(rdx, rax); // kills rbx
-  __ load_unsigned_word(rax,
-                        Address(rdx, rax,
-                                Address::times_2,
-                                arrayOopDesc::base_offset_in_bytes(T_CHAR)));
+  __ load_unsigned_short(rax,
+                         Address(rdx, rax,
+                                 Address::times_2,
+                                 arrayOopDesc::base_offset_in_bytes(T_CHAR)));
 }
 
 void TemplateTable::saload() {
@@ -675,10 +675,10 @@
   // eax: index
   // rdx: array
   index_check(rdx, rax); // kills rbx
-  __ load_signed_word(rax,
-                      Address(rdx, rax,
-                              Address::times_2,
-                              arrayOopDesc::base_offset_in_bytes(T_SHORT)));
+  __ load_signed_short(rax,
+                       Address(rdx, rax,
+                               Address::times_2,
+                               arrayOopDesc::base_offset_in_bytes(T_SHORT)));
 }
 
 void TemplateTable::iload(int n) {
@@ -2276,7 +2276,7 @@
   __ cmpl(flags, ctos);
   __ jcc(Assembler::notEqual, notChar);
   // ctos
-  __ load_unsigned_word(rax, field);
+  __ load_unsigned_short(rax, field);
   __ push(ctos);
   // Rewrite bytecode to be faster
   if (!is_static) {
@@ -2288,7 +2288,7 @@
   __ cmpl(flags, stos);
   __ jcc(Assembler::notEqual, notShort);
   // stos
-  __ load_signed_word(rax, field);
+  __ load_signed_short(rax, field);
   __ push(stos);
   // Rewrite bytecode to be faster
   if (!is_static) {
@@ -2751,10 +2751,10 @@
     __ movsbl(rax, field);
     break;
   case Bytecodes::_fast_sgetfield:
-    __ load_signed_word(rax, field);
+    __ load_signed_short(rax, field);
     break;
   case Bytecodes::_fast_cgetfield:
-    __ load_unsigned_word(rax, field);
+    __ load_unsigned_short(rax, field);
     break;
   case Bytecodes::_fast_fgetfield:
     __ movflt(xmm0, field);
@@ -3010,97 +3010,55 @@
   // profile this call
   __ profile_virtual_call(rdx, r13, r14);
 
-  __ mov(r14, rdx); // Save klassOop in r14
-
-  // Compute start of first itableOffsetEntry (which is at the end of
-  // the vtable)
-  const int base = instanceKlass::vtable_start_offset() * wordSize;
-  // Get length of vtable
-  assert(vtableEntry::size() * wordSize == 8,
-         "adjust the scaling in the code below");
-  __ movl(r13, Address(rdx,
-                       instanceKlass::vtable_length_offset() * wordSize));
-  __ lea(rdx, Address(rdx, r13, Address::times_8, base));
-
-  if (HeapWordsPerLong > 1) {
-    // Round up to align_object_offset boundary
-    __ round_to(rdx, BytesPerLong);
-  }
-
-  Label entry, search, interface_ok;
-
-  __ jmpb(entry);
-  __ bind(search);
-  __ addptr(rdx, itableOffsetEntry::size() * wordSize);
-
-  __ bind(entry);
-
-  // Check that the entry is non-null.  A null entry means that the
-  // receiver class doesn't implement the interface, and wasn't the
-  // same as the receiver class checked when the interface was
-  // resolved.
-  __ push(rdx);
-  __ movptr(rdx, Address(rdx, itableOffsetEntry::interface_offset_in_bytes()));
-  __ testptr(rdx, rdx);
-  __ jcc(Assembler::notZero, interface_ok);
+  Label no_such_interface, no_such_method;
+
+  __ lookup_interface_method(// inputs: rec. class, interface, itable index
+                             rdx, rax, rbx,
+                             // outputs: method, scan temp. reg
+                             rbx, r13,
+                             no_such_interface);
+
+  // rbx,: methodOop to call
+  // rcx: receiver
+  // Check for abstract method error
+  // Note: This should be done more efficiently via a throw_abstract_method_error
+  //       interpreter entry point and a conditional jump to it in case of a null
+  //       method.
+  __ testptr(rbx, rbx);
+  __ jcc(Assembler::zero, no_such_method);
+
+  // do the call
+  // rcx: receiver
+  // rbx,: methodOop
+  __ jump_from_interpreted(rbx, rdx);
+  __ should_not_reach_here();
+
+  // exception handling code follows...
+  // note: must restore interpreter registers to canonical
+  //       state for exception handling to work correctly!
+
+  __ bind(no_such_method);
   // throw exception
-  __ pop(rdx); // pop saved register first.
-  __ pop(rbx); // pop return address (pushed by prepare_invoke)
-  __ restore_bcp(); // r13 must be correct for exception handler (was
-                    // destroyed)
-  __ restore_locals(); // make sure locals pointer is correct as well
-                       // (was destroyed)
+  __ pop(rbx);           // pop return address (pushed by prepare_invoke)
+  __ restore_bcp();      // r13 must be correct for exception handler   (was destroyed)
+  __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
+  __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
+  // the call_VM checks for exception, so we should never return here.
+  __ should_not_reach_here();
+
+  __ bind(no_such_interface);
+  // throw exception
+  __ pop(rbx);           // pop return address (pushed by prepare_invoke)
+  __ restore_bcp();      // r13 must be correct for exception handler   (was destroyed)
+  __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
                    InterpreterRuntime::throw_IncompatibleClassChangeError));
   // the call_VM checks for exception, so we should never return here.
   __ should_not_reach_here();
-  __ bind(interface_ok);
-
-  __ pop(rdx);
-
-  __ cmpptr(rax, Address(rdx, itableOffsetEntry::interface_offset_in_bytes()));
-  __ jcc(Assembler::notEqual, search);
-
-  __ movl(rdx, Address(rdx, itableOffsetEntry::offset_offset_in_bytes()));
-
-  __ addptr(rdx, r14); // Add offset to klassOop
-  assert(itableMethodEntry::size() * wordSize == 8,
-         "adjust the scaling in the code below");
-  __ movptr(rbx, Address(rdx, rbx, Address::times_8));
-  // rbx: methodOop to call
-  // rcx: receiver
-  // Check for abstract method error
-  // Note: This should be done more efficiently via a
-  // throw_abstract_method_error interpreter entry point and a
-  // conditional jump to it in case of a null method.
-  {
-    Label L;
-    __ testptr(rbx, rbx);
-    __ jcc(Assembler::notZero, L);
-    // throw exception
-    // note: must restore interpreter registers to canonical
-    //       state for exception handling to work correctly!
-    __ pop(rbx);  // pop return address (pushed by prepare_invoke)
-    __ restore_bcp(); // r13 must be correct for exception handler
-                      // (was destroyed)
-    __ restore_locals(); // make sure locals pointer is correct as
-                         // well (was destroyed)
-    __ call_VM(noreg,
-               CAST_FROM_FN_PTR(address,
-                             InterpreterRuntime::throw_AbstractMethodError));
-    // the call_VM checks for exception, so we should never return here.
-    __ should_not_reach_here();
-    __ bind(L);
-  }
-
-  __ movptr(rcx, Address(rbx, methodOopDesc::interpreter_entry_offset()));
-
-  // do the call
-  // rcx: receiver
-  // rbx: methodOop
-  __ jump_from_interpreted(rbx, rdx);
+  return;
 }
 
+
 //-----------------------------------------------------------------------------
 // Allocation
 
--- a/src/cpu/x86/vm/vtableStubs_x86_32.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/vtableStubs_x86_32.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -34,10 +34,16 @@
 extern "C" void bad_compiled_vtable_index(JavaThread* thread, oop receiver, int index);
 #endif
 
-// used by compiler only; may use only caller saved registers rax, rbx, rcx.
-// rdx holds first int arg, rsi, rdi, rbp are callee-save & must be preserved.
-// Leave receiver in rcx; required behavior when +OptoArgsInRegisters
-// is modifed to put first oop in rcx.
+// These stubs are used by the compiler only.
+// Argument registers, which must be preserved:
+//   rcx - receiver (always first argument)
+//   rdx - second argument (if any)
+// Other registers that might be usable:
+//   rax - inline cache register (is interface for itable stub)
+//   rbx - method (used when calling out to interpreter)
+// Available now, but may become callee-save at some point:
+//   rsi, rdi
+// Note that rax and rdx are also used for return values.
 //
 VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
   const int i486_code_length = VtableStub::pd_code_size_limit(true);
@@ -94,16 +100,25 @@
   __ jmp( Address(method, methodOopDesc::from_compiled_offset()));
 
   masm->flush();
+
+  if (PrintMiscellaneous && (WizardMode || Verbose)) {
+    tty->print_cr("vtable #%d at "PTR_FORMAT"[%d] left over: %d",
+                  vtable_index, s->entry_point(),
+                  (int)(s->code_end() - s->entry_point()),
+                  (int)(s->code_end() - __ pc()));
+  }
+  guarantee(__ pc() <= s->code_end(), "overflowed buffer");
+
   s->set_exception_points(npe_addr, ame_addr);
   return s;
 }
 
 
-VtableStub* VtableStubs::create_itable_stub(int vtable_index) {
+VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   // Note well: pd_code_size_limit is the absolute minimum we can get away with.  If you
   //            add code here, bump the code stub size returned by pd_code_size_limit!
   const int i486_code_length = VtableStub::pd_code_size_limit(false);
-  VtableStub* s = new(i486_code_length) VtableStub(false, vtable_index);
+  VtableStub* s = new(i486_code_length) VtableStub(false, itable_index);
   ResourceMark rm;
   CodeBuffer cb(s->entry_point(), i486_code_length);
   MacroAssembler* masm = new MacroAssembler(&cb);
@@ -123,50 +138,19 @@
 
   // get receiver klass (also an implicit null-check)
   address npe_addr = __ pc();
-  __ movptr(rbx, Address(rcx, oopDesc::klass_offset_in_bytes()));
-
-  __ mov(rsi, rbx);   // Save klass in free register
-  // Most registers are in use, so save a few
-  __ push(rdx);
-  // compute itable entry offset (in words)
-  const int base = instanceKlass::vtable_start_offset() * wordSize;
-  assert(vtableEntry::size() * wordSize == 4, "adjust the scaling in the code below");
-  __ movl(rdx, Address(rbx, instanceKlass::vtable_length_offset() * wordSize)); // Get length of vtable
-  __ lea(rbx, Address(rbx, rdx, Address::times_ptr, base));
-  if (HeapWordsPerLong > 1) {
-    // Round up to align_object_offset boundary
-    __ round_to(rbx, BytesPerLong);
-  }
-
-  Label hit, next, entry, throw_icce;
-
-  __ jmpb(entry);
+  __ movptr(rsi, Address(rcx, oopDesc::klass_offset_in_bytes()));
 
-  __ bind(next);
-  __ addptr(rbx, itableOffsetEntry::size() * wordSize);
-
-  __ bind(entry);
-
-  // If the entry is NULL then we've reached the end of the table
-  // without finding the expected interface, so throw an exception
-  __ movptr(rdx, Address(rbx, itableOffsetEntry::interface_offset_in_bytes()));
-  __ testptr(rdx, rdx);
-  __ jcc(Assembler::zero, throw_icce);
-  __ cmpptr(rax, rdx);
-  __ jcc(Assembler::notEqual, next);
-
-  // We found a hit, move offset into rbx,
-  __ movl(rdx, Address(rbx, itableOffsetEntry::offset_offset_in_bytes()));
-
-  // Compute itableMethodEntry.
-  const int method_offset = (itableMethodEntry::size() * wordSize * vtable_index) + itableMethodEntry::method_offset_in_bytes();
+  // Most registers are in use; we'll use rax, rbx, rsi, rdi
+  // (If we need to make rsi, rdi callee-save, do a push/pop here.)
+  const Register method = rbx;
+  Label throw_icce;
 
   // Get methodOop and entrypoint for compiler
-  const Register method = rbx;
-  __ movptr(method, Address(rsi, rdx, Address::times_1, method_offset));
-
-  // Restore saved register, before possible trap.
-  __ pop(rdx);
+  __ lookup_interface_method(// inputs: rec. class, interface, itable index
+                             rsi, rax, itable_index,
+                             // outputs: method, scan temp. reg
+                             method, rdi,
+                             throw_icce);
 
   // method (rbx): methodOop
   // rcx: receiver
@@ -187,12 +171,15 @@
   __ jmp(Address(method, methodOopDesc::from_compiled_offset()));
 
   __ bind(throw_icce);
-  // Restore saved register
-  __ pop(rdx);
   __ jump(RuntimeAddress(StubRoutines::throw_IncompatibleClassChangeError_entry()));
-
   masm->flush();
 
+  if (PrintMiscellaneous && (WizardMode || Verbose)) {
+    tty->print_cr("itable #%d at "PTR_FORMAT"[%d] left over: %d",
+                  itable_index, s->entry_point(),
+                  (int)(s->code_end() - s->entry_point()),
+                  (int)(s->code_end() - __ pc()));
+  }
   guarantee(__ pc() <= s->code_end(), "overflowed buffer");
 
   s->set_exception_points(npe_addr, ame_addr);
@@ -207,7 +194,7 @@
     return (DebugVtables ? 210 : 16) + (CountCompiledCalls ? 6 : 0);
   } else {
     // Itable stub size
-    return (DebugVtables ? 144 : 64) + (CountCompiledCalls ? 6 : 0);
+    return (DebugVtables ? 256 : 66) + (CountCompiledCalls ? 6 : 0);
   }
 }
 
--- a/src/cpu/x86/vm/vtableStubs_x86_64.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/vtableStubs_x86_64.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -98,17 +98,26 @@
   __ jmp( Address(rbx, methodOopDesc::from_compiled_offset()));
 
   __ flush();
+
+  if (PrintMiscellaneous && (WizardMode || Verbose)) {
+    tty->print_cr("vtable #%d at "PTR_FORMAT"[%d] left over: %d",
+                  vtable_index, s->entry_point(),
+                  (int)(s->code_end() - s->entry_point()),
+                  (int)(s->code_end() - __ pc()));
+  }
+  guarantee(__ pc() <= s->code_end(), "overflowed buffer");
+
   s->set_exception_points(npe_addr, ame_addr);
   return s;
 }
 
 
-VtableStub* VtableStubs::create_itable_stub(int vtable_index) {
+VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   // Note well: pd_code_size_limit is the absolute minimum we can get
   // away with.  If you add code here, bump the code stub size
   // returned by pd_code_size_limit!
   const int amd64_code_length = VtableStub::pd_code_size_limit(false);
-  VtableStub* s = new(amd64_code_length) VtableStub(false, vtable_index);
+  VtableStub* s = new(amd64_code_length) VtableStub(false, itable_index);
   ResourceMark rm;
   CodeBuffer cb(s->entry_point(), amd64_code_length);
   MacroAssembler* masm = new MacroAssembler(&cb);
@@ -131,68 +140,28 @@
   // get receiver klass (also an implicit null-check)
   address npe_addr = __ pc();
 
-  __ load_klass(rbx, j_rarg0);
+  // Most registers are in use; we'll use rax, rbx, r10, r11
+  // (various calling sequences use r[cd]x, r[sd]i, r[89]; stay away from them)
+  __ load_klass(r10, j_rarg0);
 
   // If we take a trap while this arg is on the stack we will not
   // be able to walk the stack properly. This is not an issue except
   // when there are mistakes in this assembly code that could generate
   // a spurious fault. Ask me how I know...
 
-  __ push(j_rarg1);     // Most registers are in use, so save one
-
-  // compute itable entry offset (in words)
-  const int base = instanceKlass::vtable_start_offset() * wordSize;
-  assert(vtableEntry::size() * wordSize == 8,
-         "adjust the scaling in the code below");
-  // Get length of vtable
-  __ movl(j_rarg1,
-          Address(rbx, instanceKlass::vtable_length_offset() * wordSize));
-  __ lea(rbx, Address(rbx, j_rarg1, Address::times_8, base));
-
-  if (HeapWordsPerLong > 1) {
-    // Round up to align_object_offset boundary
-    __ round_to(rbx, BytesPerLong);
-  }
-  Label hit, next, entry, throw_icce;
-
-  __ jmpb(entry);
-
-  __ bind(next);
-  __ addptr(rbx, itableOffsetEntry::size() * wordSize);
-
-  __ bind(entry);
-
-  // If the entry is NULL then we've reached the end of the table
-  // without finding the expected interface, so throw an exception
-  __ movptr(j_rarg1, Address(rbx, itableOffsetEntry::interface_offset_in_bytes()));
-  __ testptr(j_rarg1, j_rarg1);
-  __ jcc(Assembler::zero, throw_icce);
-  __ cmpptr(rax, j_rarg1);
-  __ jccb(Assembler::notEqual, next);
-
-  // We found a hit, move offset into j_rarg1
-  __ movl(j_rarg1, Address(rbx, itableOffsetEntry::offset_offset_in_bytes()));
-
-  // Compute itableMethodEntry
-  const int method_offset =
-    (itableMethodEntry::size() * wordSize * vtable_index) +
-    itableMethodEntry::method_offset_in_bytes();
+  const Register method = rbx;
+  Label throw_icce;
 
   // Get methodOop and entrypoint for compiler
-
-  // Get klass pointer again
-  __ load_klass(rax, j_rarg0);
-
-  const Register method = rbx;
-  __ movptr(method, Address(rax, j_rarg1, Address::times_1, method_offset));
-
-  // Restore saved register, before possible trap.
-  __ pop(j_rarg1);
+  __ lookup_interface_method(// inputs: rec. class, interface, itable index
+                             r10, rax, itable_index,
+                             // outputs: method, scan temp. reg
+                             method, r11,
+                             throw_icce);
 
   // method (rbx): methodOop
   // j_rarg0: receiver
 
-
 #ifdef ASSERT
   if (DebugVtables) {
     Label L2;
@@ -211,12 +180,16 @@
   __ jmp(Address(method, methodOopDesc::from_compiled_offset()));
 
   __ bind(throw_icce);
-  // Restore saved register
-  __ pop(j_rarg1);
   __ jump(RuntimeAddress(StubRoutines::throw_IncompatibleClassChangeError_entry()));
 
   __ flush();
 
+  if (PrintMiscellaneous && (WizardMode || Verbose)) {
+    tty->print_cr("itable #%d at "PTR_FORMAT"[%d] left over: %d",
+                  itable_index, s->entry_point(),
+                  (int)(s->code_end() - s->entry_point()),
+                  (int)(s->code_end() - __ pc()));
+  }
   guarantee(__ pc() <= s->code_end(), "overflowed buffer");
 
   s->set_exception_points(npe_addr, ame_addr);
@@ -230,7 +203,7 @@
            (UseCompressedOops ? 16 : 0);  // 1 leaq can be 3 bytes + 1 long
   } else {
     // Itable stub size
-    return (DebugVtables ? 636 : 72) + (CountCompiledCalls ? 13 : 0) +
+    return (DebugVtables ? 512 : 72) + (CountCompiledCalls ? 13 : 0) +
            (UseCompressedOops ? 32 : 0);  // 2 leaqs
   }
 }
--- a/src/cpu/x86/vm/x86_32.ad	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/x86_32.ad	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -3126,14 +3126,12 @@
 
   enc_class movq_ld(regXD dst, memory mem) %{
     MacroAssembler _masm(&cbuf);
-    Address madr = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp);
-    __ movq(as_XMMRegister($dst$$reg), madr);
+    __ movq($dst$$XMMRegister, $mem$$Address);
   %}
 
   enc_class movq_st(memory mem, regXD src) %{
     MacroAssembler _masm(&cbuf);
-    Address madr = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp);
-    __ movq(madr, as_XMMRegister($src$$reg));
+    __ movq($mem$$Address, $src$$XMMRegister);
   %}
 
   enc_class pshufd_8x8(regX dst, regX src) %{
@@ -3751,8 +3749,8 @@
     masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL);
 
     // Load first characters
-    masm.load_unsigned_word(rcx, Address(rbx, 0));
-    masm.load_unsigned_word(rdi, Address(rax, 0));
+    masm.load_unsigned_short(rcx, Address(rbx, 0));
+    masm.load_unsigned_short(rdi, Address(rax, 0));
 
     // Compare first characters
     masm.subl(rcx, rdi);
@@ -3782,8 +3780,8 @@
 
     // Compare the rest of the characters
     masm.bind(WHILE_HEAD_LABEL);
-    masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0));
-    masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0));
+    masm.load_unsigned_short(rcx, Address(rbx, rsi, Address::times_2, 0));
+    masm.load_unsigned_short(rdi, Address(rax, rsi, Address::times_2, 0));
     masm.subl(rcx, rdi);
     masm.jcc(Assembler::notZero, POP_LABEL);
     masm.incrementl(rsi);
@@ -3840,8 +3838,8 @@
     masm.jcc(Assembler::zero, COMPARE_LOOP_HDR);
 
     // Compare 2-byte "tail" at end of arrays
-    masm.load_unsigned_word(tmp1Reg, Address(ary1Reg, resultReg, Address::times_4, base_offset));
-    masm.load_unsigned_word(tmp2Reg, Address(ary2Reg, resultReg, Address::times_4, base_offset));
+    masm.load_unsigned_short(tmp1Reg, Address(ary1Reg, resultReg, Address::times_4, base_offset));
+    masm.load_unsigned_short(tmp2Reg, Address(ary2Reg, resultReg, Address::times_4, base_offset));
     masm.cmpl(tmp1Reg, tmp2Reg);
     masm.jcc(Assembler::notEqual, FALSE_LABEL);
     masm.testl(resultReg, resultReg);
@@ -6396,21 +6394,94 @@
   match(Set dst (LoadB mem));
 
   ins_cost(125);
-  format %{ "MOVSX8 $dst,$mem" %}
-  opcode(0xBE, 0x0F);
-  ins_encode( OpcS, OpcP, RegMem(dst,mem));
-  ins_pipe( ialu_reg_mem );
-%}
-
-// Load Byte (8bit UNsigned)
-instruct loadUB(xRegI dst, memory mem, immI_255 bytemask) %{
-  match(Set dst (AndI (LoadB mem) bytemask));
+  format %{ "MOVSX8 $dst,$mem\t# byte" %}
+
+  ins_encode %{
+    __ movsbl($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Byte (8bit signed) into Long Register
+instruct loadB2L(eRegL dst, memory mem) %{
+  match(Set dst (ConvI2L (LoadB mem)));
+
+  ins_cost(375);
+  format %{ "MOVSX8 $dst.lo,$mem\t# byte -> long\n\t"
+            "MOV    $dst.hi,$dst.lo\n\t"
+            "SAR    $dst.hi,7" %}
+
+  ins_encode %{
+    __ movsbl($dst$$Register, $mem$$Address);
+    __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
+    __ sarl(HIGH_FROM_LOW($dst$$Register), 7); // 24+1 MSB are already signed extended.
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Unsigned Byte (8bit UNsigned)
+instruct loadUB(xRegI dst, memory mem) %{
+  match(Set dst (LoadUB mem));
 
   ins_cost(125);
-  format %{ "MOVZX8 $dst,$mem" %}
-  opcode(0xB6, 0x0F);
-  ins_encode( OpcS, OpcP, RegMem(dst,mem));
-  ins_pipe( ialu_reg_mem );
+  format %{ "MOVZX8 $dst,$mem\t# ubyte -> int" %}
+
+  ins_encode %{
+    __ movzbl($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Unsigned Byte (8 bit UNsigned) into Long Register
+instruct loadUB2L(eRegL dst, memory mem)
+%{
+  match(Set dst (ConvI2L (LoadUB mem)));
+
+  ins_cost(250);
+  format %{ "MOVZX8 $dst.lo,$mem\t# ubyte -> long\n\t"
+            "XOR    $dst.hi,$dst.hi" %}
+
+  ins_encode %{
+    __ movzbl($dst$$Register, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Short (16bit signed)
+instruct loadS(eRegI dst, memory mem) %{
+  match(Set dst (LoadS mem));
+
+  ins_cost(125);
+  format %{ "MOVSX  $dst,$mem\t# short" %}
+
+  ins_encode %{
+    __ movswl($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Short (16bit signed) into Long Register
+instruct loadS2L(eRegL dst, memory mem) %{
+  match(Set dst (ConvI2L (LoadS mem)));
+
+  ins_cost(375);
+  format %{ "MOVSX  $dst.lo,$mem\t# short -> long\n\t"
+            "MOV    $dst.hi,$dst.lo\n\t"
+            "SAR    $dst.hi,15" %}
+
+  ins_encode %{
+    __ movswl($dst$$Register, $mem$$Address);
+    __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
+    __ sarl(HIGH_FROM_LOW($dst$$Register), 15); // 16+1 MSB are already signed extended.
+  %}
+
+  ins_pipe(ialu_reg_mem);
 %}
 
 // Load Unsigned Short/Char (16bit unsigned)
@@ -6418,10 +6489,30 @@
   match(Set dst (LoadUS mem));
 
   ins_cost(125);
-  format %{ "MOVZX  $dst,$mem" %}
-  opcode(0xB7, 0x0F);
-  ins_encode( OpcS, OpcP, RegMem(dst,mem));
-  ins_pipe( ialu_reg_mem );
+  format %{ "MOVZX  $dst,$mem\t# ushort/char -> int" %}
+
+  ins_encode %{
+    __ movzwl($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Unsigned Short/Char (16 bit UNsigned) into Long Register
+instruct loadUS2L(eRegL dst, memory mem)
+%{
+  match(Set dst (ConvI2L (LoadUS mem)));
+
+  ins_cost(250);
+  format %{ "MOVZX  $dst.lo,$mem\t# ushort/char -> long\n\t"
+            "XOR    $dst.hi,$dst.hi" %}
+
+  ins_encode %{
+    __ movzwl($dst$$Register, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
+  %}
+
+  ins_pipe(ialu_reg_mem);
 %}
 
 // Load Integer
@@ -6429,10 +6520,47 @@
   match(Set dst (LoadI mem));
 
   ins_cost(125);
-  format %{ "MOV    $dst,$mem" %}
-  opcode(0x8B);
-  ins_encode( OpcP, RegMem(dst,mem));
-  ins_pipe( ialu_reg_mem );
+  format %{ "MOV    $dst,$mem\t# int" %}
+
+  ins_encode %{
+    __ movl($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Integer into Long Register
+instruct loadI2L(eRegL dst, memory mem) %{
+  match(Set dst (ConvI2L (LoadI mem)));
+
+  ins_cost(375);
+  format %{ "MOV    $dst.lo,$mem\t# int -> long\n\t"
+            "MOV    $dst.hi,$dst.lo\n\t"
+            "SAR    $dst.hi,31" %}
+
+  ins_encode %{
+    __ movl($dst$$Register, $mem$$Address);
+    __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
+    __ sarl(HIGH_FROM_LOW($dst$$Register), 31);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Unsigned Integer into Long Register
+instruct loadUI2L(eRegL dst, memory mem) %{
+  match(Set dst (LoadUI2L mem));
+
+  ins_cost(250);
+  format %{ "MOV    $dst.lo,$mem\t# uint -> long\n\t"
+            "XOR    $dst.hi,$dst.hi" %}
+
+  ins_encode %{
+    __ movl($dst$$Register, $mem$$Address);
+    __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
+  %}
+
+  ins_pipe(ialu_reg_mem);
 %}
 
 // Load Long.  Cannot clobber address while loading, so restrict address
@@ -6442,11 +6570,17 @@
   match(Set dst (LoadL mem));
 
   ins_cost(250);
-  format %{ "MOV    $dst.lo,$mem\n\t"
+  format %{ "MOV    $dst.lo,$mem\t# long\n\t"
             "MOV    $dst.hi,$mem+4" %}
-  opcode(0x8B, 0x8B);
-  ins_encode( OpcP, RegMem(dst,mem), OpcS, RegMem_Hi(dst,mem));
-  ins_pipe( ialu_reg_long_mem );
+
+  ins_encode %{
+    Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, false);
+    Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, false);
+    __ movl($dst$$Register, Amemlo);
+    __ movl(HIGH_FROM_LOW($dst$$Register), Amemhi);
+  %}
+
+  ins_pipe(ialu_reg_long_mem);
 %}
 
 // Volatile Load Long.  Must be atomic, so do 64-bit FILD
@@ -6521,17 +6655,6 @@
   ins_pipe( ialu_reg_mem );
 %}
 
-// Load Short (16bit signed)
-instruct loadS(eRegI dst, memory mem) %{
-  match(Set dst (LoadS mem));
-
-  ins_cost(125);
-  format %{ "MOVSX  $dst,$mem" %}
-  opcode(0xBF, 0x0F);
-  ins_encode( OpcS, OpcP, RegMem(dst,mem));
-  ins_pipe( ialu_reg_mem );
-%}
-
 // Load Double
 instruct loadD(regD dst, memory mem) %{
   predicate(UseSSE<=1);
@@ -7957,7 +8080,7 @@
     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
     if( os::is_MP() )
       __ lock();
-    __ cmpxchg8(Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp));
+    __ cmpxchg8($mem$$Address);
     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
   %}
   ins_pipe( pipe_cmpxchg );
@@ -11467,6 +11590,7 @@
 instruct convI2L_reg( eRegL dst, eRegI src, eFlagsReg cr) %{
   match(Set dst (ConvI2L src));
   effect(KILL cr);
+  ins_cost(375);
   format %{ "MOV    $dst.lo,$src\n\t"
             "MOV    $dst.hi,$src\n\t"
             "SAR    $dst.hi,31" %}
@@ -11478,6 +11602,7 @@
 instruct convI2L_reg_zex(eRegL dst, eRegI src, immL_32bits mask, eFlagsReg flags ) %{
   match(Set dst (AndL (ConvI2L src) mask) );
   effect( KILL flags );
+  ins_cost(250);
   format %{ "MOV    $dst.lo,$src\n\t"
             "XOR    $dst.hi,$dst.hi" %}
   opcode(0x33); // XOR
@@ -11489,6 +11614,7 @@
 instruct zerox_long(eRegL dst, eRegL src, immL_32bits mask, eFlagsReg flags ) %{
   match(Set dst (AndL src mask) );
   effect( KILL flags );
+  ins_cost(250);
   format %{ "MOV    $dst.lo,$src.lo\n\t"
             "XOR    $dst.hi,$dst.hi\n\t" %}
   opcode(0x33); // XOR
--- a/src/cpu/x86/vm/x86_64.ad	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/cpu/x86/vm/x86_64.ad	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -3462,14 +3462,12 @@
 
   enc_class movq_ld(regD dst, memory mem) %{
     MacroAssembler _masm(&cbuf);
-    Address madr = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp);
-    __ movq(as_XMMRegister($dst$$reg), madr);
+    __ movq($dst$$XMMRegister, $mem$$Address);
   %}
 
   enc_class movq_st(memory mem, regD src) %{
     MacroAssembler _masm(&cbuf);
-    Address madr = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp);
-    __ movq(madr, as_XMMRegister($src$$reg));
+    __ movq($mem$$Address, $src$$XMMRegister);
   %}
 
   enc_class pshufd_8x8(regF dst, regF src) %{
@@ -3765,8 +3763,8 @@
     masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL);
 
     // Load first characters
-    masm.load_unsigned_word(rcx, Address(rbx, 0));
-    masm.load_unsigned_word(rdi, Address(rax, 0));
+    masm.load_unsigned_short(rcx, Address(rbx, 0));
+    masm.load_unsigned_short(rdi, Address(rax, 0));
 
     // Compare first characters
     masm.subl(rcx, rdi);
@@ -3796,8 +3794,8 @@
 
     // Compare the rest of the characters
     masm.bind(WHILE_HEAD_LABEL);
-    masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0));
-    masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0));
+    masm.load_unsigned_short(rcx, Address(rbx, rsi, Address::times_2, 0));
+    masm.load_unsigned_short(rdi, Address(rax, rsi, Address::times_2, 0));
     masm.subl(rcx, rdi);
     masm.jcc(Assembler::notZero, POP_LABEL);
     masm.increment(rsi);
@@ -3854,8 +3852,8 @@
     masm.jcc(Assembler::zero, COMPARE_LOOP_HDR);
 
     // Compare 2-byte "tail" at end of arrays
-    masm.load_unsigned_word(tmp1Reg, Address(ary1Reg, resultReg, Address::times_4, base_offset));
-    masm.load_unsigned_word(tmp2Reg, Address(ary2Reg, resultReg, Address::times_4, base_offset));
+    masm.load_unsigned_short(tmp1Reg, Address(ary1Reg, resultReg, Address::times_4, base_offset));
+    masm.load_unsigned_short(tmp2Reg, Address(ary2Reg, resultReg, Address::times_4, base_offset));
     masm.cmpl(tmp1Reg, tmp2Reg);
     masm.jcc(Assembler::notEqual, FALSE_LABEL);
     masm.testl(resultReg, resultReg);
@@ -6031,70 +6029,88 @@
 
   ins_cost(125);
   format %{ "movsbl  $dst, $mem\t# byte" %}
-  opcode(0x0F, 0xBE);
-  ins_encode(REX_reg_mem(dst, mem), OpcP, OpcS, reg_mem(dst, mem));
+
+  ins_encode %{
+    __ movsbl($dst$$Register, $mem$$Address);
+  %}
+
   ins_pipe(ialu_reg_mem);
 %}
 
-// Load Byte (8 bit signed) into long
-// instruct loadB2L(rRegL dst, memory mem)
-// %{
-//   match(Set dst (ConvI2L (LoadB mem)));
-
-//   ins_cost(125);
-//   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
-//   opcode(0x0F, 0xBE);
-//   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, OpcS, reg_mem(dst, mem));
-//   ins_pipe(ialu_reg_mem);
-// %}
-
-// Load Byte (8 bit UNsigned)
-instruct loadUB(rRegI dst, memory mem, immI_255 bytemask)
-%{
-  match(Set dst (AndI (LoadB mem) bytemask));
+// Load Byte (8 bit signed) into Long Register
+instruct loadB2L(rRegL dst, memory mem)
+%{
+  match(Set dst (ConvI2L (LoadB mem)));
+
+  ins_cost(125);
+  format %{ "movsbq  $dst, $mem\t# byte -> long" %}
+
+  ins_encode %{
+    __ movsbq($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Unsigned Byte (8 bit UNsigned)
+instruct loadUB(rRegI dst, memory mem)
+%{
+  match(Set dst (LoadUB mem));
 
   ins_cost(125);
   format %{ "movzbl  $dst, $mem\t# ubyte" %}
-  opcode(0x0F, 0xB6);
-  ins_encode(REX_reg_mem(dst, mem), OpcP, OpcS, reg_mem(dst, mem));
+
+  ins_encode %{
+    __ movzbl($dst$$Register, $mem$$Address);
+  %}
+
   ins_pipe(ialu_reg_mem);
 %}
 
-// Load Byte (8 bit UNsigned) into long
-// instruct loadUB2L(rRegL dst, memory mem, immI_255 bytemask)
-// %{
-//   match(Set dst (ConvI2L (AndI (LoadB mem) bytemask)));
-
-//   ins_cost(125);
-//   format %{ "movzbl  $dst, $mem\t# ubyte -> long" %}
-//   opcode(0x0F, 0xB6);
-//   ins_encode(REX_reg_mem(dst, mem), OpcP, OpcS, reg_mem(dst, mem));
-//   ins_pipe(ialu_reg_mem);
-// %}
+// Load Unsigned Byte (8 bit UNsigned) into Long Register
+instruct loadUB2L(rRegL dst, memory mem)
+%{
+  match(Set dst (ConvI2L (LoadUB mem)));
+
+  ins_cost(125);
+  format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
+
+  ins_encode %{
+    __ movzbq($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
 
 // Load Short (16 bit signed)
 instruct loadS(rRegI dst, memory mem)
 %{
   match(Set dst (LoadS mem));
 
-  ins_cost(125); // XXX
+  ins_cost(125);
   format %{ "movswl $dst, $mem\t# short" %}
-  opcode(0x0F, 0xBF);
-  ins_encode(REX_reg_mem(dst, mem), OpcP, OpcS, reg_mem(dst, mem));
+
+  ins_encode %{
+    __ movswl($dst$$Register, $mem$$Address);
+  %}
+
   ins_pipe(ialu_reg_mem);
 %}
 
-// Load Short (16 bit signed) into long
-// instruct loadS2L(rRegL dst, memory mem)
-// %{
-//   match(Set dst (ConvI2L (LoadS mem)));
-
-//   ins_cost(125); // XXX
-//   format %{ "movswq $dst, $mem\t# short -> long" %}
-//   opcode(0x0F, 0xBF);
-//   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, OpcS, reg_mem(dst, mem));
-//   ins_pipe(ialu_reg_mem);
-// %}
+// Load Short (16 bit signed) into Long Register
+instruct loadS2L(rRegL dst, memory mem)
+%{
+  match(Set dst (ConvI2L (LoadS mem)));
+
+  ins_cost(125);
+  format %{ "movswq $dst, $mem\t# short -> long" %}
+
+  ins_encode %{
+    __ movswq($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
 
 // Load Unsigned Short/Char (16 bit UNsigned)
 instruct loadUS(rRegI dst, memory mem)
@@ -6103,32 +6119,71 @@
 
   ins_cost(125);
   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
-  opcode(0x0F, 0xB7);
-  ins_encode(REX_reg_mem(dst, mem), OpcP, OpcS, reg_mem(dst, mem));
+
+  ins_encode %{
+    __ movzwl($dst$$Register, $mem$$Address);
+  %}
+
   ins_pipe(ialu_reg_mem);
 %}
 
-// Load Unsigned Short/Char (16 bit UNsigned) into long
-// instruct loadUS2L(rRegL dst, memory mem)
-// %{
-//   match(Set dst (ConvI2L (LoadUS mem)));
-
-//   ins_cost(125);
-//   format %{ "movzwl  $dst, $mem\t# ushort/char -> long" %}
-//   opcode(0x0F, 0xB7);
-//   ins_encode(REX_reg_mem(dst, mem), OpcP, OpcS, reg_mem(dst, mem));
-//   ins_pipe(ialu_reg_mem);
-// %}
+// Load Unsigned Short/Char (16 bit UNsigned) into Long Register
+instruct loadUS2L(rRegL dst, memory mem)
+%{
+  match(Set dst (ConvI2L (LoadUS mem)));
+
+  ins_cost(125);
+  format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
+
+  ins_encode %{
+    __ movzwq($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
 
 // Load Integer
 instruct loadI(rRegI dst, memory mem)
 %{
   match(Set dst (LoadI mem));
 
-  ins_cost(125); // XXX
+  ins_cost(125);
   format %{ "movl    $dst, $mem\t# int" %}
-  opcode(0x8B);
-  ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
+
+  ins_encode %{
+    __ movl($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Integer into Long Register
+instruct loadI2L(rRegL dst, memory mem)
+%{
+  match(Set dst (ConvI2L (LoadI mem)));
+
+  ins_cost(125);
+  format %{ "movslq  $dst, $mem\t# int -> long" %}
+
+  ins_encode %{
+    __ movslq($dst$$Register, $mem$$Address);
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+// Load Unsigned Integer into Long Register
+instruct loadUI2L(rRegL dst, memory mem)
+%{
+  match(Set dst (LoadUI2L mem));
+
+  ins_cost(125);
+  format %{ "movl    $dst, $mem\t# uint -> long" %}
+
+  ins_encode %{
+    __ movl($dst$$Register, $mem$$Address);
+  %}
+
   ins_pipe(ialu_reg_mem);
 %}
 
@@ -6137,10 +6192,13 @@
 %{
   match(Set dst (LoadL mem));
 
-  ins_cost(125); // XXX
+  ins_cost(125);
   format %{ "movq    $dst, $mem\t# long" %}
-  opcode(0x8B);
-  ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
+
+  ins_encode %{
+    __ movq($dst$$Register, $mem$$Address);
+  %}
+
   ins_pipe(ialu_reg_mem); // XXX
 %}
 
@@ -10804,16 +10862,6 @@
 //   ins_pipe(ialu_reg_reg);
 // %}
 
-instruct convI2L_reg_mem(rRegL dst, memory src)
-%{
-  match(Set dst (ConvI2L (LoadI src)));
-
-  format %{ "movslq  $dst, $src\t# i2l" %}
-  opcode(0x63); // needs REX.W
-  ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst,src));
-  ins_pipe(ialu_reg_mem);
-%}
-
 // Zero-extend convert int to long
 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
 %{
--- a/src/os/linux/vm/os_linux.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/os/linux/vm/os_linux.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os/solaris/vm/os_solaris.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/os/solaris/vm/os_solaris.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os/windows/vm/os_windows.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/os/windows/vm/os_windows.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/linux_x86/vm/os_linux_x86.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/os_cpu/linux_x86/vm/os_linux_x86.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/solaris_x86/vm/solaris_x86_32.il	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/os_cpu/solaris_x86/vm/solaris_x86_32.il	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2003-2007 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/os_cpu/solaris_x86/vm/solaris_x86_64.il	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/os_cpu/solaris_x86/vm/solaris_x86_64.il	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2004-2007 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2004-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/adlc/adlparse.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/adlc/adlparse.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/adlc/dict2.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/adlc/dict2.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2002 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/adlc/forms.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/adlc/forms.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -70,6 +70,7 @@
   else return (_iter <_cur-1 ? _names[++_iter] : NULL);
 }
 const char  *NameList::current() { return (_iter < _cur ? _names[_iter] : NULL); }
+const char  *NameList::peek(int skip) { return (_iter + skip < _cur ? _names[_iter + skip] : NULL); }
 
 // Return 'true' if current entry is signal
 bool  NameList::current_is_signal() {
@@ -248,11 +249,13 @@
 // True if 'opType', an ideal name, loads or stores.
 Form::DataType Form::is_load_from_memory(const char *opType) const {
   if( strcmp(opType,"LoadB")==0 )  return Form::idealB;
+  if( strcmp(opType,"LoadUB")==0 )  return Form::idealB;
   if( strcmp(opType,"LoadUS")==0 )  return Form::idealC;
   if( strcmp(opType,"LoadD")==0 )  return Form::idealD;
   if( strcmp(opType,"LoadD_unaligned")==0 )  return Form::idealD;
   if( strcmp(opType,"LoadF")==0 )  return Form::idealF;
   if( strcmp(opType,"LoadI")==0 )  return Form::idealI;
+  if( strcmp(opType,"LoadUI2L")==0 )  return Form::idealI;
   if( strcmp(opType,"LoadKlass")==0 )  return Form::idealP;
   if( strcmp(opType,"LoadNKlass")==0 ) return Form::idealN;
   if( strcmp(opType,"LoadL")==0 )  return Form::idealL;
--- a/src/share/vm/adlc/forms.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/adlc/forms.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -342,6 +342,7 @@
   void  reset();                   // Reset iteration
   const char *iter();              // after reset(), first element : else next
   const char *current();           // return current element in iteration.
+  const char *peek(int skip = 1);  // returns element + skip in iteration if there is one
 
   bool  current_is_signal();       // Return 'true' if current entry is signal
   bool  is_signal(const char *entry); // Return true if entry is a signal
--- a/src/share/vm/adlc/formssel.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/adlc/formssel.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -3310,8 +3310,8 @@
   static const char *needs_ideal_memory_list[] = {
     "StoreI","StoreL","StoreP","StoreN","StoreD","StoreF" ,
     "StoreB","StoreC","Store" ,"StoreFP",
-    "LoadI" ,"LoadL", "LoadP" ,"LoadN", "LoadD" ,"LoadF"  ,
-    "LoadB" ,"LoadUS" ,"LoadS" ,"Load"   ,
+    "LoadI", "LoadUI2L", "LoadL", "LoadP" ,"LoadN", "LoadD" ,"LoadF"  ,
+    "LoadB" , "LoadUB", "LoadUS" ,"LoadS" ,"Load"   ,
     "Store4I","Store2I","Store2L","Store2D","Store4F","Store2F","Store16B",
     "Store8B","Store4B","Store8C","Store4C","Store2C",
     "Load4I" ,"Load2I" ,"Load2L" ,"Load2D" ,"Load4F" ,"Load2F" ,"Load16B" ,
@@ -3431,10 +3431,16 @@
     const InstructForm *form2_inst = form2 ? form2->is_instruction() : NULL;
     const char *name_left  = mRule2->_lChild ? mRule2->_lChild->_opType : NULL;
     const char *name_right = mRule2->_rChild ? mRule2->_rChild->_opType : NULL;
+    DataType data_type = Form::none;
+    if (form->is_operand()) {
+      // Make sure the loadX matches the type of the reg
+      data_type = form->ideal_to_Reg_type(form->is_operand()->ideal_type(globals));
+    }
     // Detect reg vs (loadX memory)
     if( form->is_cisc_reg(globals)
         && form2_inst
-        && (is_load_from_memory(mRule2->_opType) != Form::none) // reg vs. (load memory)
+        && data_type != Form::none
+        && (is_load_from_memory(mRule2->_opType) == data_type) // reg vs. (load memory)
         && (name_left != NULL)       // NOT (load)
         && (name_right == NULL) ) {  // NOT (load memory foo)
       const Form *form2_left = name_left ? globals[name_left] : NULL;
--- a/src/share/vm/adlc/output_c.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/adlc/output_c.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -2139,8 +2139,59 @@
         // A subfield variable, '$$' prefix
         emit_field( rep_var );
       } else {
-        // A replacement variable, '$' prefix
-        emit_rep_var( rep_var );
+        if (_strings_to_emit.peek() != NULL &&
+            strcmp(_strings_to_emit.peek(), "$Address") == 0) {
+          fprintf(_fp, "Address::make_raw(");
+
+          emit_rep_var( rep_var );
+          fprintf(_fp,"->base(ra_,this,idx%d), ", _operand_idx);
+
+          _reg_status = LITERAL_ACCESSED;
+          emit_rep_var( rep_var );
+          fprintf(_fp,"->index(ra_,this,idx%d), ", _operand_idx);
+
+          _reg_status = LITERAL_ACCESSED;
+          emit_rep_var( rep_var );
+          fprintf(_fp,"->scale(), ");
+
+          _reg_status = LITERAL_ACCESSED;
+          emit_rep_var( rep_var );
+          Form::DataType stack_type = _operand ? _operand->is_user_name_for_sReg() : Form::none;
+          if( _operand  && _operand_idx==0 && stack_type != Form::none ) {
+            fprintf(_fp,"->disp(ra_,this,0), ");
+          } else {
+            fprintf(_fp,"->disp(ra_,this,idx%d), ", _operand_idx);
+          }
+
+          _reg_status = LITERAL_ACCESSED;
+          emit_rep_var( rep_var );
+          fprintf(_fp,"->disp_is_oop())");
+
+          // skip trailing $Address
+          _strings_to_emit.iter();
+        } else {
+          // A replacement variable, '$' prefix
+          const char* next = _strings_to_emit.peek();
+          const char* next2 = _strings_to_emit.peek(2);
+          if (next != NULL && next2 != NULL && strcmp(next2, "$Register") == 0 &&
+              (strcmp(next, "$base") == 0 || strcmp(next, "$index") == 0)) {
+            // handle $rev_var$$base$$Register and $rev_var$$index$$Register by
+            // producing as_Register(opnd_array(#)->base(ra_,this,idx1)).
+            fprintf(_fp, "as_Register(");
+            // emit the operand reference
+            emit_rep_var( rep_var );
+            rep_var = _strings_to_emit.iter();
+            assert(strcmp(rep_var, "$base") == 0 || strcmp(rep_var, "$index") == 0, "bad pattern");
+            // handle base or index
+            emit_field(rep_var);
+            rep_var = _strings_to_emit.iter();
+            assert(strcmp(rep_var, "$Register") == 0, "bad pattern");
+            // close up the parens
+            fprintf(_fp, ")");
+          } else {
+            emit_rep_var( rep_var );
+          }
+        }
       } // end replacement and/or subfield
     }
   }
--- a/src/share/vm/asm/assembler.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/asm/assembler.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -239,6 +239,78 @@
   }
 }
 
+struct DelayedConstant {
+  typedef void (*value_fn_t)();
+  BasicType type;
+  intptr_t value;
+  value_fn_t value_fn;
+  // This limit of 20 is generous for initial uses.
+  // The limit needs to be large enough to store the field offsets
+  // into classes which do not have statically fixed layouts.
+  // (Initial use is for method handle object offsets.)
+  // Look for uses of "delayed_value" in the source code
+  // and make sure this number is generous enough to handle all of them.
+  enum { DC_LIMIT = 20 };
+  static DelayedConstant delayed_constants[DC_LIMIT];
+  static DelayedConstant* add(BasicType type, value_fn_t value_fn);
+  bool match(BasicType t, value_fn_t cfn) {
+    return type == t && value_fn == cfn;
+  }
+  static void update_all();
+};
+
+DelayedConstant DelayedConstant::delayed_constants[DC_LIMIT];
+// Default C structure initialization rules have the following effect here:
+// = { { (BasicType)0, (intptr_t)NULL }, ... };
+
+DelayedConstant* DelayedConstant::add(BasicType type,
+                                      DelayedConstant::value_fn_t cfn) {
+  for (int i = 0; i < DC_LIMIT; i++) {
+    DelayedConstant* dcon = &delayed_constants[i];
+    if (dcon->match(type, cfn))
+      return dcon;
+    if (dcon->value_fn == NULL) {
+      // (cmpxchg not because this is multi-threaded but because I'm paranoid)
+      if (Atomic::cmpxchg_ptr(CAST_FROM_FN_PTR(void*, cfn), &dcon->value_fn, NULL) == NULL) {
+        dcon->type = type;
+        return dcon;
+      }
+    }
+  }
+  // If this assert is hit (in pre-integration testing!) then re-evaluate
+  // the comment on the definition of DC_LIMIT.
+  guarantee(false, "too many delayed constants");
+  return NULL;
+}
+
+void DelayedConstant::update_all() {
+  for (int i = 0; i < DC_LIMIT; i++) {
+    DelayedConstant* dcon = &delayed_constants[i];
+    if (dcon->value_fn != NULL && dcon->value == 0) {
+      typedef int     (*int_fn_t)();
+      typedef address (*address_fn_t)();
+      switch (dcon->type) {
+      case T_INT:     dcon->value = (intptr_t) ((int_fn_t)    dcon->value_fn)(); break;
+      case T_ADDRESS: dcon->value = (intptr_t) ((address_fn_t)dcon->value_fn)(); break;
+      }
+    }
+  }
+}
+
+intptr_t* AbstractAssembler::delayed_value_addr(int(*value_fn)()) {
+  DelayedConstant* dcon = DelayedConstant::add(T_INT, (DelayedConstant::value_fn_t) value_fn);
+  return &dcon->value;
+}
+intptr_t* AbstractAssembler::delayed_value_addr(address(*value_fn)()) {
+  DelayedConstant* dcon = DelayedConstant::add(T_ADDRESS, (DelayedConstant::value_fn_t) value_fn);
+  return &dcon->value;
+}
+void AbstractAssembler::update_delayed_values() {
+  DelayedConstant::update_all();
+}
+
+
+
 
 void AbstractAssembler::block_comment(const char* comment) {
   if (sect() == CodeBuffer::SECT_INSTS) {
--- a/src/share/vm/asm/assembler.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/asm/assembler.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -140,6 +140,28 @@
   }
 };
 
+// A union type for code which has to assemble both constant and
+// non-constant operands, when the distinction cannot be made
+// statically.
+class RegisterConstant VALUE_OBJ_CLASS_SPEC {
+ private:
+  Register _r;
+  intptr_t _c;
+
+ public:
+  RegisterConstant(): _r(noreg), _c(0) {}
+  RegisterConstant(Register r): _r(r), _c(0) {}
+  RegisterConstant(intptr_t c): _r(noreg), _c(c) {}
+
+  Register as_register() const { assert(is_register(),""); return _r; }
+  intptr_t as_constant() const { assert(is_constant(),""); return _c; }
+
+  Register register_or_noreg() const { return _r; }
+  intptr_t constant_or_zero() const  { return _c; }
+
+  bool is_register() const { return _r != noreg; }
+  bool is_constant() const { return _r == noreg; }
+};
 
 // The Abstract Assembler: Pure assembler doing NO optimizations on the
 // instruction level; i.e., what you write is what you get.
@@ -280,6 +302,26 @@
   inline address address_constant(Label& L);
   inline address address_table_constant(GrowableArray<Label*> label);
 
+  // Bootstrapping aid to cope with delayed determination of constants.
+  // Returns a static address which will eventually contain the constant.
+  // The value zero (NULL) stands instead of a constant which is still uncomputed.
+  // Thus, the eventual value of the constant must not be zero.
+  // This is fine, since this is designed for embedding object field
+  // offsets in code which must be generated before the object class is loaded.
+  // Field offsets are never zero, since an object's header (mark word)
+  // is located at offset zero.
+  RegisterConstant delayed_value(int(*value_fn)(), Register tmp, int offset = 0) {
+    return delayed_value(delayed_value_addr(value_fn), tmp, offset);
+  }
+  RegisterConstant delayed_value(address(*value_fn)(), Register tmp, int offset = 0) {
+    return delayed_value(delayed_value_addr(value_fn), tmp, offset);
+  }
+  virtual RegisterConstant delayed_value(intptr_t* delayed_value_addr, Register tmp, int offset) = 0;
+  // Last overloading is platform-dependent; look in assembler_<arch>.cpp.
+  static intptr_t* delayed_value_addr(int(*constant_fn)());
+  static intptr_t* delayed_value_addr(address(*constant_fn)());
+  static void update_delayed_values();
+
   // Bang stack to trigger StackOverflowError at a safe location
   // implementation delegates to machine-specific bang_stack_with_offset
   void generate_stack_overflow_check( int frame_size_in_bytes );
--- a/src/share/vm/asm/codeBuffer.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/asm/codeBuffer.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/c1/c1_LIRGenerator.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/c1/c1_LIRGenerator.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/c1/c1_Optimizer.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/c1/c1_Optimizer.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/c1/c1_Runtime1.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/c1/c1_Runtime1.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/classfile/classFileParser.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/classfile/classFileParser.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/classfile/javaClasses.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/classfile/javaClasses.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/classfile/javaClasses.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/classfile/javaClasses.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/classfile/systemDictionary.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/classfile/systemDictionary.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/classfile/systemDictionary.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/classfile/systemDictionary.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/classfile/vmSymbols.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/classfile/vmSymbols.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/collectionSetChooser.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/collectionSetChooser.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -133,14 +133,12 @@
       _co_tracker.update(false);
 
       if (G1SmoothConcRefine) {
-        start_vtime_sec = os::elapsedVTime();
         prev_buffer_num = curr_buffer_num;
-
         _sts.leave();
         os::sleep(Thread::current(), (jlong) _interval_ms, false);
         _sts.join();
+        start_vtime_sec = os::elapsedVTime();
       }
-
       n_logs++;
     }
     // Make sure we harvest the PYA, if any.
--- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -420,6 +420,10 @@
 
   _has_overflown(false),
   _concurrent(false),
+  _has_aborted(false),
+  _restart_for_overflow(false),
+  _concurrent_marking_in_progress(false),
+  _should_gray_objects(false),
 
   // _verbose_level set below
 
--- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -107,7 +107,7 @@
       if (PrintGC) {
         gclog_or_tty->date_stamp(PrintGCDateStamps);
         gclog_or_tty->stamp(PrintGCTimeStamps);
-        tty->print_cr("[GC concurrent-mark-start]");
+        gclog_or_tty->print_cr("[GC concurrent-mark-start]");
       }
 
       if (!g1_policy->in_young_gc_mode()) {
@@ -320,8 +320,6 @@
   set_in_progress();
   clear_started();
   if (TraceConcurrentMark) gclog_or_tty->print_cr("CM-starting");
-
-  return;
 }
 
 // Note: this method, although exported by the ConcurrentMarkSweepThread,
--- a/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -78,8 +78,8 @@
 
 void DirtyCardQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock,
                                    int max_completed_queue,
-                                   Mutex* lock) {
-  PtrQueueSet::initialize(cbl_mon, fl_lock, max_completed_queue);
+                                   Mutex* lock, PtrQueueSet* fl_owner) {
+  PtrQueueSet::initialize(cbl_mon, fl_lock, max_completed_queue, fl_owner);
   set_buffer_size(DCQBarrierQueueBufferSize);
   set_process_completed_threshold(DCQBarrierProcessCompletedThreshold);
 
--- a/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -88,7 +88,7 @@
 
   void initialize(Monitor* cbl_mon, Mutex* fl_lock,
                   int max_completed_queue = 0,
-                  Mutex* lock = NULL);
+                  Mutex* lock = NULL, PtrQueueSet* fl_owner = NULL);
 
   // The number of parallel ids that can be claimed to allow collector or
   // mutator threads to do card-processing work.
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -136,6 +136,14 @@
   int calls() { return _calls; }
 };
 
+class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {
+public:
+  bool do_card_ptr(jbyte* card_ptr, int worker_i) {
+    *card_ptr = CardTableModRefBS::dirty_card_val();
+    return true;
+  }
+};
+
 YoungList::YoungList(G1CollectedHeap* g1h)
   : _g1h(g1h), _head(NULL),
     _scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL),
@@ -812,6 +820,40 @@
   }
 };
 
+class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
+  G1CollectedHeap*   _g1h;
+  UpdateRSOopClosure _cl;
+  int                _worker_i;
+public:
+  RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, int worker_i = 0) :
+    _cl(g1->g1_rem_set()->as_HRInto_G1RemSet(), worker_i),
+    _worker_i(worker_i),
+    _g1h(g1)
+  { }
+  bool doHeapRegion(HeapRegion* r) {
+    if (!r->continuesHumongous()) {
+      _cl.set_from(r);
+      r->oop_iterate(&_cl);
+    }
+    return false;
+  }
+};
+
+class ParRebuildRSTask: public AbstractGangTask {
+  G1CollectedHeap* _g1;
+public:
+  ParRebuildRSTask(G1CollectedHeap* g1)
+    : AbstractGangTask("ParRebuildRSTask"),
+      _g1(g1)
+  { }
+
+  void work(int i) {
+    RebuildRSOutOfRegionClosure rebuild_rs(_g1, i);
+    _g1->heap_region_par_iterate_chunked(&rebuild_rs, i,
+                                         HeapRegion::RebuildRSClaimValue);
+  }
+};
+
 void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs,
                                     size_t word_size) {
   ResourceMark rm;
@@ -918,24 +960,35 @@
 
     reset_gc_time_stamp();
     // Since everything potentially moved, we will clear all remembered
-    // sets, and clear all cards.  Later we will also cards in the used
-    // portion of the heap after the resizing (which could be a shrinking.)
-    // We will also reset the GC time stamps of the regions.
+    // sets, and clear all cards.  Later we will rebuild remebered
+    // sets. We will also reset the GC time stamps of the regions.
     PostMCRemSetClearClosure rs_clear(mr_bs());
     heap_region_iterate(&rs_clear);
 
     // Resize the heap if necessary.
     resize_if_necessary_after_full_collection(full ? 0 : word_size);
 
-    // Since everything potentially moved, we will clear all remembered
-    // sets, but also dirty all cards corresponding to used regions.
-    PostMCRemSetInvalidateClosure rs_invalidate(mr_bs());
-    heap_region_iterate(&rs_invalidate);
     if (_cg1r->use_cache()) {
       _cg1r->clear_and_record_card_counts();
       _cg1r->clear_hot_cache();
     }
 
+    // Rebuild remembered sets of all regions.
+    if (ParallelGCThreads > 0) {
+      ParRebuildRSTask rebuild_rs_task(this);
+      assert(check_heap_region_claim_values(
+             HeapRegion::InitialClaimValue), "sanity check");
+      set_par_threads(workers()->total_workers());
+      workers()->run_task(&rebuild_rs_task);
+      set_par_threads(0);
+      assert(check_heap_region_claim_values(
+             HeapRegion::RebuildRSClaimValue), "sanity check");
+      reset_heap_region_claim_values();
+    } else {
+      RebuildRSOutOfRegionClosure rebuild_rs(this);
+      heap_region_iterate(&rebuild_rs);
+    }
+
     if (PrintGC) {
       print_size_transition(gclog_or_tty, g1h_prev_used, used(), capacity());
     }
@@ -961,7 +1014,8 @@
     // dirtied, so this should abandon those logs, and set "do_traversal"
     // to true.
     concurrent_g1_refine()->set_pya_restart();
-
+    assert(!G1DeferredRSUpdate
+           || (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any");
     assert(regions_accounted_for(), "Region leakage!");
   }
 
@@ -1466,6 +1520,13 @@
                                                   G1DirtyCardQueueMax,
                                                   Shared_DirtyCardQ_lock);
   }
+  if (G1DeferredRSUpdate) {
+    dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,
+                                      DirtyCardQ_FL_lock,
+                                      0,
+                                      Shared_DirtyCardQ_lock,
+                                      &JavaThread::dirty_card_queue_set());
+  }
   // In case we're keeping closure specialization stats, initialize those
   // counts and that mechanism.
   SpecializationStats::clear();
@@ -2316,7 +2377,6 @@
 void
 G1CollectedHeap::checkConcurrentMark() {
     VerifyMarkedObjsClosure verifycl(this);
-    doConcurrentMark();
     //    MutexLockerEx x(getMarkBitMapLock(),
     //              Mutex::_no_safepoint_check_flag);
     object_iterate(&verifycl);
@@ -2493,7 +2553,7 @@
 
     guarantee(_in_cset_fast_test == NULL, "invariant");
     guarantee(_in_cset_fast_test_base == NULL, "invariant");
-    _in_cset_fast_test_length = n_regions();
+    _in_cset_fast_test_length = max_regions();
     _in_cset_fast_test_base =
                              NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);
     memset(_in_cset_fast_test_base, false,
@@ -2918,27 +2978,51 @@
   }
 };
 
-class RecreateRSetEntriesClosure: public OopClosure {
+class UpdateRSetImmediate : public OopsInHeapRegionClosure {
 private:
   G1CollectedHeap* _g1;
   G1RemSet* _g1_rem_set;
-  HeapRegion* _from;
 public:
-  RecreateRSetEntriesClosure(G1CollectedHeap* g1, HeapRegion* from) :
-    _g1(g1), _g1_rem_set(g1->g1_rem_set()), _from(from)
-  {}
+  UpdateRSetImmediate(G1CollectedHeap* g1) :
+    _g1(g1), _g1_rem_set(g1->g1_rem_set()) {}
 
   void do_oop(narrowOop* p) {
     guarantee(false, "NYI");
   }
   void do_oop(oop* p) {
     assert(_from->is_in_reserved(p), "paranoia");
-    if (*p != NULL) {
-      _g1_rem_set->write_ref(_from, p);
+    if (*p != NULL && !_from->is_survivor()) {
+      _g1_rem_set->par_write_ref(_from, p, 0);
     }
   }
 };
 
+class UpdateRSetDeferred : public OopsInHeapRegionClosure {
+private:
+  G1CollectedHeap* _g1;
+  DirtyCardQueue *_dcq;
+  CardTableModRefBS* _ct_bs;
+
+public:
+  UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
+    _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {}
+
+  void do_oop(narrowOop* p) {
+    guarantee(false, "NYI");
+  }
+  void do_oop(oop* p) {
+    assert(_from->is_in_reserved(p), "paranoia");
+    if (!_from->is_in_reserved(*p) && !_from->is_survivor()) {
+      size_t card_index = _ct_bs->index_for(p);
+      if (_ct_bs->mark_card_deferred(card_index)) {
+        _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
+      }
+    }
+  }
+};
+
+
+
 class RemoveSelfPointerClosure: public ObjectClosure {
 private:
   G1CollectedHeap* _g1;
@@ -2946,11 +3030,11 @@
   HeapRegion* _hr;
   size_t _prev_marked_bytes;
   size_t _next_marked_bytes;
+  OopsInHeapRegionClosure *_cl;
 public:
-  RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr) :
-    _g1(g1), _cm(_g1->concurrent_mark()), _hr(hr),
-    _prev_marked_bytes(0), _next_marked_bytes(0)
-  {}
+  RemoveSelfPointerClosure(G1CollectedHeap* g1, OopsInHeapRegionClosure* cl) :
+    _g1(g1), _cm(_g1->concurrent_mark()),  _prev_marked_bytes(0),
+    _next_marked_bytes(0), _cl(cl) {}
 
   size_t prev_marked_bytes() { return _prev_marked_bytes; }
   size_t next_marked_bytes() { return _next_marked_bytes; }
@@ -2988,8 +3072,7 @@
       // that, if evacuation fails, we might have remembered set
       // entries missing given that we skipped cards on the
       // collection set. So, we'll recreate such entries now.
-      RecreateRSetEntriesClosure cl(_g1, _hr);
-      obj->oop_iterate(&cl);
+      obj->oop_iterate(_cl);
       assert(_cm->isPrevMarked(obj), "Should be marked!");
     } else {
       // The object has been either evacuated or is dead. Fill it with a
@@ -3002,14 +3085,23 @@
 };
 
 void G1CollectedHeap::remove_self_forwarding_pointers() {
+  UpdateRSetImmediate immediate_update(_g1h);
+  DirtyCardQueue dcq(&_g1h->dirty_card_queue_set());
+  UpdateRSetDeferred deferred_update(_g1h, &dcq);
+  OopsInHeapRegionClosure *cl;
+  if (G1DeferredRSUpdate) {
+    cl = &deferred_update;
+  } else {
+    cl = &immediate_update;
+  }
   HeapRegion* cur = g1_policy()->collection_set();
-
   while (cur != NULL) {
     assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!");
 
+    RemoveSelfPointerClosure rspc(_g1h, cl);
     if (cur->evacuation_failed()) {
-      RemoveSelfPointerClosure rspc(_g1h, cur);
       assert(cur->in_collection_set(), "bad CS");
+      cl->set_region(cur);
       cur->object_iterate(&rspc);
 
       // A number of manipulations to make the TAMS be the current top,
@@ -3518,6 +3610,9 @@
 protected:
   G1CollectedHeap* _g1h;
   RefToScanQueue*  _refs;
+  DirtyCardQueue   _dcq;
+  CardTableModRefBS* _ct_bs;
+  G1RemSet* _g1_rem;
 
   typedef GrowableArray<oop*> OverflowQueue;
   OverflowQueue* _overflowed_refs;
@@ -3559,10 +3654,32 @@
 
   void   add_to_undo_waste(size_t waste)         { _undo_waste += waste; }
 
+  DirtyCardQueue& dirty_card_queue()             { return _dcq;  }
+  CardTableModRefBS* ctbs()                      { return _ct_bs; }
+
+  void immediate_rs_update(HeapRegion* from, oop* p, int tid) {
+    _g1_rem->par_write_ref(from, p, tid);
+  }
+
+  void deferred_rs_update(HeapRegion* from, oop* p, int tid) {
+    // If the new value of the field points to the same region or
+    // is the to-space, we don't need to include it in the Rset updates.
+    if (!from->is_in_reserved(*p) && !from->is_survivor()) {
+      size_t card_index = ctbs()->index_for(p);
+      // If the card hasn't been added to the buffer, do it.
+      if (ctbs()->mark_card_deferred(card_index)) {
+        dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
+      }
+    }
+  }
+
 public:
   G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
     : _g1h(g1h),
       _refs(g1h->task_queue(queue_num)),
+      _dcq(&g1h->dirty_card_queue_set()),
+      _ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
+      _g1_rem(g1h->g1_rem_set()),
       _hash_seed(17), _queue_num(queue_num),
       _term_attempts(0),
       _age_table(false),
@@ -3640,6 +3757,14 @@
   int refs_to_scan()                             { return refs()->size();                 }
   int overflowed_refs_to_scan()                  { return overflowed_refs()->length();    }
 
+  void update_rs(HeapRegion* from, oop* p, int tid) {
+    if (G1DeferredRSUpdate) {
+      deferred_rs_update(from, p, tid);
+    } else {
+      immediate_rs_update(from, p, tid);
+    }
+  }
+
   HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) {
 
     HeapWord* obj = NULL;
@@ -3808,7 +3933,6 @@
   }
 };
 
-
 G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
   _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
   _par_scan_state(par_scan_state) { }
@@ -3834,7 +3958,7 @@
       assert(obj == *p, "the value of *p should not have changed");
       _par_scan_state->push_on_queue(p);
     } else {
-      _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
+      _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
     }
   }
 }
@@ -3972,13 +4096,13 @@
     }
     // When scanning the RS, we only care about objs in CS.
     if (barrier == G1BarrierRS) {
-      _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
+      _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
     }
   }
 
   // When scanning moved objs, must look at all oops.
   if (barrier == G1BarrierEvac && obj != NULL) {
-    _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
+    _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
   }
 
   if (do_gen_barrier && obj != NULL) {
@@ -4127,6 +4251,7 @@
     G1ParScanExtRootClosure         only_scan_root_cl(_g1h, &pss);
     G1ParScanPermClosure            only_scan_perm_cl(_g1h, &pss);
     G1ParScanHeapRSClosure          only_scan_heap_rs_cl(_g1h, &pss);
+
     G1ParScanAndMarkExtRootClosure  scan_mark_root_cl(_g1h, &pss);
     G1ParScanAndMarkPermClosure     scan_mark_perm_cl(_g1h, &pss);
     G1ParScanAndMarkHeapRSClosure   scan_mark_heap_rs_cl(_g1h, &pss);
@@ -4382,7 +4507,6 @@
   g1_rem_set()->prepare_for_oops_into_collection_set_do();
   concurrent_g1_refine()->set_use_cache(false);
   int n_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1);
-
   set_par_threads(n_workers);
   G1ParTask g1_par_task(this, n_workers, _task_queues);
 
@@ -4390,8 +4514,9 @@
 
   change_strong_roots_parity();  // In preparation for parallel strong roots.
   rem_set()->prepare_for_younger_refs_iterate(true);
+
+  assert(dirty_card_queue_set().completed_buffers_num() == 0, "Should be empty");
   double start_par = os::elapsedTime();
-
   if (ParallelGCThreads > 0) {
     // The individual threads will set their evac-failure closures.
     workers()->run_task(&g1_par_task);
@@ -4411,8 +4536,8 @@
     G1KeepAliveClosure keep_alive(this);
     JNIHandles::weak_oops_do(&is_alive, &keep_alive);
   }
-
   g1_rem_set()->cleanup_after_oops_into_collection_set_do();
+
   concurrent_g1_refine()->set_use_cache(true);
 
   finalize_for_evac_failure();
@@ -4423,7 +4548,6 @@
 
   if (evacuation_failed()) {
     remove_self_forwarding_pointers();
-
     if (PrintGCDetails) {
       gclog_or_tty->print(" (evacuation failed)");
     } else if (PrintGC) {
@@ -4431,6 +4555,14 @@
     }
   }
 
+  if (G1DeferredRSUpdate) {
+    RedirtyLoggedCardTableEntryFastClosure redirty;
+    dirty_card_queue_set().set_closure(&redirty);
+    dirty_card_queue_set().apply_closure_to_all_completed_buffers();
+    JavaThread::dirty_card_queue_set().merge_bufferlists(&dirty_card_queue_set());
+    assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
+  }
+
   COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
 }
 
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -457,6 +457,10 @@
   // And it's mod ref barrier set, used to track updates for the above.
   ModRefBarrierSet* _mr_bs;
 
+  // A set of cards that cover the objects for which the Rsets should be updated
+  // concurrently after the collection.
+  DirtyCardQueueSet _dirty_card_queue_set;
+
   // The Heap Region Rem Set Iterator.
   HeapRegionRemSetIterator** _rem_set_iterator;
 
@@ -666,6 +670,9 @@
 
   RefToScanQueue *task_queue(int i);
 
+  // A set of cards where updates happened during the GC
+  DirtyCardQueueSet& dirty_card_queue_set() { return _dirty_card_queue_set; }
+
   // Create a G1CollectedHeap with the specified policy.
   // Must call the initialize method afterwards.
   // May not return if something goes wrong.
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/g1MMUTracker.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1MMUTracker.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/g1RemSet.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1RemSet.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -105,33 +105,6 @@
   _g1->heap_region_iterate(&rc);
 }
 
-class UpdateRSOopClosure: public OopClosure {
-  HeapRegion* _from;
-  HRInto_G1RemSet* _rs;
-  int _worker_i;
-public:
-  UpdateRSOopClosure(HRInto_G1RemSet* rs, int worker_i = 0) :
-    _from(NULL), _rs(rs), _worker_i(worker_i) {
-    guarantee(_rs != NULL, "Requires an HRIntoG1RemSet");
-  }
-
-  void set_from(HeapRegion* from) {
-    assert(from != NULL, "from region must be non-NULL");
-    _from = from;
-  }
-
-  virtual void do_oop(narrowOop* p) {
-    guarantee(false, "NYI");
-  }
-  virtual void do_oop(oop* p) {
-    assert(_from != NULL, "from region must be non-NULL");
-    _rs->par_write_ref(_from, p, _worker_i);
-  }
-  // Override: this closure is idempotent.
-  //  bool idempotent() { return true; }
-  bool apply_to_weak_ref_discovered_field() { return true; }
-};
-
 class UpdateRSOutOfRegionClosure: public HeapRegionClosure {
   G1CollectedHeap*    _g1h;
   ModRefBarrierSet*   _mr_bs;
@@ -177,11 +150,19 @@
     _cards_scanned(NULL), _total_cards_scanned(0)
 {
   _seq_task = new SubTasksDone(NumSeqTasks);
-  _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<oop*>*, ParallelGCThreads);
+  guarantee(n_workers() > 0, "There should be some workers");
+  _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<oop*>*, n_workers());
+  for (uint i = 0; i < n_workers(); i++) {
+    _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<oop*>(8192,true);
+  }
 }
 
 HRInto_G1RemSet::~HRInto_G1RemSet() {
   delete _seq_task;
+  for (uint i = 0; i < n_workers(); i++) {
+    delete _new_refs[i];
+  }
+  FREE_C_HEAP_ARRAY(GrowableArray<oop*>*, _new_refs);
 }
 
 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) {
@@ -281,8 +262,9 @@
         if (!_ct_bs->is_card_claimed(card_index) &&
             !_ct_bs->is_card_dirty(card_index)) {
           assert(_ct_bs->is_card_clean(card_index) ||
-                 _ct_bs->is_card_claimed(card_index),
-                 "Card is either dirty, clean, or claimed");
+                 _ct_bs->is_card_claimed(card_index) ||
+                 _ct_bs->is_card_deferred(card_index),
+                 "Card is either clean, claimed or deferred");
           if (_ct_bs->claim_card(card_index))
             scanCard(card_index, card_region);
         }
@@ -338,14 +320,12 @@
 
   _g1p->record_scan_rs_start_time(worker_i, rs_time_start * 1000.0);
   _g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0);
-  if (ParallelGCThreads > 0) {
-    // In this case, we called scanNewRefsRS and recorded the corresponding
-    // time.
-    double scan_new_refs_time_ms = _g1p->get_scan_new_refs_time(worker_i);
-    if (scan_new_refs_time_ms > 0.0) {
-      closure_app_time_ms += scan_new_refs_time_ms;
-    }
+
+  double scan_new_refs_time_ms = _g1p->get_scan_new_refs_time(worker_i);
+  if (scan_new_refs_time_ms > 0.0) {
+    closure_app_time_ms += scan_new_refs_time_ms;
   }
+
   _g1p->record_obj_copy_time(worker_i, closure_app_time_ms);
 }
 
@@ -469,8 +449,8 @@
   double scan_new_refs_start_sec = os::elapsedTime();
   G1CollectedHeap* g1h = G1CollectedHeap::heap();
   CardTableModRefBS* ct_bs = (CardTableModRefBS*) (g1h->barrier_set());
-  while (_new_refs[worker_i]->is_nonempty()) {
-    oop* p = _new_refs[worker_i]->pop();
+  for (int i = 0; i < _new_refs[worker_i]->length(); i++) {
+    oop* p = _new_refs[worker_i]->at(i);
     oop obj = *p;
     // *p was in the collection set when p was pushed on "_new_refs", but
     // another thread may have processed this location from an RS, so it
@@ -480,10 +460,6 @@
       HeapRegion* r = g1h->heap_region_containing(p);
 
       DEBUG_ONLY(HeapRegion* to = g1h->heap_region_containing(obj));
-      assert(ParallelGCThreads > 1
-             || to->rem_set()->contains_reference(p),
-             "Invariant: pushed after being added."
-             "(Not reliable in parallel code.)");
       oc->set_region(r);
       // If "p" has already been processed concurrently, this is
       // idempotent.
@@ -538,8 +514,8 @@
     }
   } else {
     assert(worker_i == 0, "invariant");
-
     updateRS(0);
+    scanNewRefsRS(oc, 0);
     scanRS(oc, 0);
   }
 }
@@ -559,11 +535,7 @@
   assert(!_par_traversal_in_progress, "Invariant between iterations.");
   if (ParallelGCThreads > 0) {
     set_par_traversal(true);
-    int n_workers = _g1->workers()->total_workers();
-    _seq_task->set_par_threads(n_workers);
-    for (uint i = 0; i < ParallelGCThreads; i++)
-      _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<oop*>(8192,true);
-
+    _seq_task->set_par_threads((int)n_workers());
     if (cg1r->do_traversal()) {
       updateRS(0);
       // Have to do this again after updaters
@@ -587,6 +559,53 @@
   }
 };
 
+class UpdateRSetOopsIntoCSImmediate : public OopClosure {
+  G1CollectedHeap* _g1;
+public:
+  UpdateRSetOopsIntoCSImmediate(G1CollectedHeap* g1) : _g1(g1) { }
+  virtual void do_oop(narrowOop* p) {
+    guarantee(false, "NYI");
+  }
+  virtual void do_oop(oop* p) {
+    HeapRegion* to = _g1->heap_region_containing(*p);
+    if (to->in_collection_set()) {
+      if (to->rem_set()->add_reference(p, 0)) {
+        _g1->schedule_popular_region_evac(to);
+      }
+    }
+  }
+};
+
+class UpdateRSetOopsIntoCSDeferred : public OopClosure {
+  G1CollectedHeap* _g1;
+  CardTableModRefBS* _ct_bs;
+  DirtyCardQueue* _dcq;
+public:
+  UpdateRSetOopsIntoCSDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
+    _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) { }
+  virtual void do_oop(narrowOop* p) {
+    guarantee(false, "NYI");
+  }
+  virtual void do_oop(oop* p) {
+    oop obj = *p;
+    if (_g1->obj_in_cs(obj)) {
+      size_t card_index = _ct_bs->index_for(p);
+      if (_ct_bs->mark_card_deferred(card_index)) {
+        _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
+      }
+    }
+  }
+};
+
+void HRInto_G1RemSet::new_refs_iterate(OopClosure* cl) {
+  for (size_t i = 0; i < n_workers(); i++) {
+    for (int j = 0; j < _new_refs[i]->length(); j++) {
+      oop* p = _new_refs[i]->at(j);
+      cl->do_oop(p);
+    }
+  }
+}
+
 void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() {
   guarantee( _cards_scanned != NULL, "invariant" );
   _total_cards_scanned = 0;
@@ -609,11 +628,25 @@
     if (cg1r->do_traversal()) {
       cg1r->cg1rThread()->set_do_traversal(false);
     }
-    for (uint i = 0; i < ParallelGCThreads; i++) {
-      delete _new_refs[i];
-    }
     set_par_traversal(false);
   }
+
+  if (_g1->evacuation_failed()) {
+    // Restore remembered sets for the regions pointing into
+    // the collection set.
+    if (G1DeferredRSUpdate) {
+      DirtyCardQueue dcq(&_g1->dirty_card_queue_set());
+      UpdateRSetOopsIntoCSDeferred deferred_update(_g1, &dcq);
+      new_refs_iterate(&deferred_update);
+    } else {
+      UpdateRSetOopsIntoCSImmediate immediate_update(_g1);
+      new_refs_iterate(&immediate_update);
+    }
+  }
+  for (uint i = 0; i < n_workers(); i++) {
+    _new_refs[i]->clear();
+  }
+
   assert(!_par_traversal_in_progress, "Invariant between iterations.");
 }
 
@@ -683,7 +716,8 @@
   bool doHeapRegion(HeapRegion* r) {
     if (!r->in_collection_set() &&
         !r->continuesHumongous() &&
-        !r->is_young()) {
+        !r->is_young() &&
+        !r->is_survivor()) {
       _update_rs_oop_cl.set_from(r);
       UpdateRSObjectClosure update_rs_obj_cl(&_update_rs_oop_cl);
 
@@ -820,7 +854,7 @@
   // before all the cards on the region are dirtied. This is unlikely,
   // and it doesn't happen often, but it can happen. So, the extra
   // check below filters out those cards.
-  if (r->is_young()) {
+  if (r->is_young() || r->is_survivor()) {
     return;
   }
   // While we are processing RSet buffers during the collection, we
--- a/src/share/vm/gc_implementation/g1/g1RemSet.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1RemSet.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -155,6 +155,7 @@
   bool _par_traversal_in_progress;
   void set_par_traversal(bool b);
   GrowableArray<oop*>** _new_refs;
+  void new_refs_iterate(OopClosure* cl);
 
 public:
   // This is called to reset dual hash tables after the gc pause
@@ -214,3 +215,27 @@
   int n() { return _n; };
   HeapWord* start_first() { return _start_first; }
 };
+
+class UpdateRSOopClosure: public OopClosure {
+  HeapRegion* _from;
+  HRInto_G1RemSet* _rs;
+  int _worker_i;
+public:
+  UpdateRSOopClosure(HRInto_G1RemSet* rs, int worker_i = 0) :
+    _from(NULL), _rs(rs), _worker_i(worker_i) {
+    guarantee(_rs != NULL, "Requires an HRIntoG1RemSet");
+  }
+
+  void set_from(HeapRegion* from) {
+    assert(from != NULL, "from region must be non-NULL");
+    _from = from;
+  }
+
+  virtual void do_oop(narrowOop* p);
+  virtual void do_oop(oop* p);
+
+  // Override: this closure is idempotent.
+  //  bool idempotent() { return true; }
+  bool apply_to_weak_ref_discovered_field() { return true; }
+};
+
--- a/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -31,24 +31,7 @@
 }
 
 inline void HRInto_G1RemSet::write_ref_nv(HeapRegion* from, oop* p) {
-  oop obj = *p;
-  assert(from != NULL && from->is_in_reserved(p),
-         "p is not in a from");
-  HeapRegion* to = _g1->heap_region_containing(obj);
-  if (from != to && to != NULL) {
-    if (!to->popular() && !from->is_survivor()) {
-#if G1_REM_SET_LOGGING
-      gclog_or_tty->print_cr("Adding " PTR_FORMAT " (" PTR_FORMAT ") to RS"
-                             " for region [" PTR_FORMAT ", " PTR_FORMAT ")",
-                             p, obj,
-                             to->bottom(), to->end());
-#endif
-      assert(to->rem_set() != NULL, "Need per-region 'into' remsets.");
-      if (to->rem_set()->add_reference(p)) {
-        _g1->schedule_popular_region_evac(to);
-      }
-    }
-  }
+  par_write_ref(from, p, 0);
 }
 
 inline void HRInto_G1RemSet::write_ref(HeapRegion* from, oop* p) {
@@ -82,7 +65,22 @@
   HeapRegion* to = _g1->heap_region_containing(obj);
   // The test below could be optimized by applying a bit op to to and from.
   if (to != NULL && from != NULL && from != to) {
-    if (!to->popular() && !from->is_survivor()) {
+    bool update_delayed = false;
+    // There is a tricky infinite loop if we keep pushing
+    // self forwarding pointers onto our _new_refs list.
+    // The _par_traversal_in_progress flag is true during the collection pause,
+    // false during the evacuation failure handing.
+    if (_par_traversal_in_progress &&
+        to->in_collection_set() && !self_forwarded(obj)) {
+      _new_refs[tid]->push(p);
+      // Deferred updates to the Cset are either discarded (in the normal case),
+      // or processed (if an evacuation failure occurs) at the end
+      // of the collection.
+      // See HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do().
+      update_delayed = true;
+    }
+
+    if (!to->popular() && !update_delayed) {
 #if G1_REM_SET_LOGGING
       gclog_or_tty->print_cr("Adding " PTR_FORMAT " (" PTR_FORMAT ") to RS"
                              " for region [" PTR_FORMAT ", " PTR_FORMAT ")",
@@ -94,11 +92,14 @@
         _g1->schedule_popular_region_evac(to);
       }
     }
-    // There is a tricky infinite loop if we keep pushing
-    // self forwarding pointers onto our _new_refs list.
-    if (_par_traversal_in_progress &&
-        to->in_collection_set() && !self_forwarded(obj)) {
-      _new_refs[tid]->push(p);
-    }
   }
 }
+
+inline void UpdateRSOopClosure::do_oop(narrowOop* p) {
+  guarantee(false, "NYI");
+}
+
+inline void UpdateRSOopClosure::do_oop(oop* p) {
+  assert(_from != NULL, "from region must be non-NULL");
+  _rs->par_write_ref(_from, p, _worker_i);
+}
--- a/src/share/vm/gc_implementation/g1/g1_globals.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -172,6 +172,9 @@
   develop(bool, G1RSBarrierUseQueue, true,                                  \
           "If true, use queueing RS barrier")                               \
                                                                             \
+  develop(bool, G1DeferredRSUpdate, true,                                   \
+          "If true, use deferred RS updates")                               \
+                                                                            \
   develop(bool, G1RSLogCheckCardTable, false,                               \
           "If true, verify that no dirty cards remain after RS log "        \
           "processing.")                                                    \
--- a/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/heapRegion.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -318,7 +318,8 @@
     FinalCountClaimValue  = 1,
     NoteEndClaimValue     = 2,
     ScrubRemSetClaimValue = 3,
-    ParVerifyClaimValue   = 4
+    ParVerifyClaimValue   = 4,
+    RebuildRSClaimValue   = 5
   };
 
   // Concurrent refinement requires contiguous heap regions (in which TLABs
--- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/ptrQueue.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/ptrQueue.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -91,15 +91,17 @@
   _n_completed_buffers(0),
   _process_completed_threshold(0), _process_completed(false),
   _buf_free_list(NULL), _buf_free_list_sz(0)
-{}
+{
+  _fl_owner = this;
+}
 
 void** PtrQueueSet::allocate_buffer() {
   assert(_sz > 0, "Didn't set a buffer size.");
-  MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag);
-  if (_buf_free_list != NULL) {
-    void** res = _buf_free_list;
-    _buf_free_list = (void**)_buf_free_list[0];
-    _buf_free_list_sz--;
+  MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag);
+  if (_fl_owner->_buf_free_list != NULL) {
+    void** res = _fl_owner->_buf_free_list;
+    _fl_owner->_buf_free_list = (void**)_fl_owner->_buf_free_list[0];
+    _fl_owner->_buf_free_list_sz--;
     // Just override the next pointer with NULL, just in case we scan this part
     // of the buffer.
     res[0] = NULL;
@@ -111,10 +113,10 @@
 
 void PtrQueueSet::deallocate_buffer(void** buf) {
   assert(_sz > 0, "Didn't set a buffer size.");
-  MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag);
-  buf[0] = (void*)_buf_free_list;
-  _buf_free_list = buf;
-  _buf_free_list_sz++;
+  MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag);
+  buf[0] = (void*)_fl_owner->_buf_free_list;
+  _fl_owner->_buf_free_list = buf;
+  _fl_owner->_buf_free_list_sz++;
 }
 
 void PtrQueueSet::reduce_free_list() {
@@ -207,3 +209,58 @@
 void PtrQueueSet::set_process_completed_threshold(size_t sz) {
   _process_completed_threshold = sz;
 }
+
+// Merge lists of buffers. Notify waiting threads if the length of the list
+// exceeds threshold. The source queue is emptied as a result. The queues
+// must share the monitor.
+void PtrQueueSet::merge_bufferlists(PtrQueueSet *src) {
+  assert(_cbl_mon == src->_cbl_mon, "Should share the same lock");
+  MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);
+  if (_completed_buffers_tail == NULL) {
+    assert(_completed_buffers_head == NULL, "Well-formedness");
+    _completed_buffers_head = src->_completed_buffers_head;
+    _completed_buffers_tail = src->_completed_buffers_tail;
+  } else {
+    assert(_completed_buffers_head != NULL, "Well formedness");
+    if (src->_completed_buffers_head != NULL) {
+      _completed_buffers_tail->next = src->_completed_buffers_head;
+      _completed_buffers_tail = src->_completed_buffers_tail;
+    }
+  }
+  _n_completed_buffers += src->_n_completed_buffers;
+
+  src->_n_completed_buffers = 0;
+  src->_completed_buffers_head = NULL;
+  src->_completed_buffers_tail = NULL;
+
+  assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL ||
+         _completed_buffers_head != NULL && _completed_buffers_tail != NULL,
+         "Sanity");
+
+  if (!_process_completed &&
+      _n_completed_buffers >= _process_completed_threshold) {
+    _process_completed = true;
+    if (_notify_when_complete)
+      _cbl_mon->notify_all();
+  }
+}
+
+// Merge free lists of the two queues. The free list of the source
+// queue is emptied as a result. The queues must share the same
+// mutex that guards free lists.
+void PtrQueueSet::merge_freelists(PtrQueueSet* src) {
+  assert(_fl_lock == src->_fl_lock, "Should share the same lock");
+  MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag);
+  if (_buf_free_list != NULL) {
+    void **p = _buf_free_list;
+    while (*p != NULL) {
+      p = (void**)*p;
+    }
+    *p = src->_buf_free_list;
+  } else {
+    _buf_free_list = src->_buf_free_list;
+  }
+  _buf_free_list_sz += src->_buf_free_list_sz;
+  src->_buf_free_list = NULL;
+  src->_buf_free_list_sz = 0;
+}
--- a/src/share/vm/gc_implementation/g1/ptrQueue.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/ptrQueue.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -155,6 +155,9 @@
   Mutex* _fl_lock;
   void** _buf_free_list;
   size_t _buf_free_list_sz;
+  // Queue set can share a freelist. The _fl_owner variable
+  // specifies the owner. It is set to "this" by default.
+  PtrQueueSet* _fl_owner;
 
   // The size of all buffers in the set.
   size_t _sz;
@@ -188,10 +191,13 @@
   // Because of init-order concerns, we can't pass these as constructor
   // arguments.
   void initialize(Monitor* cbl_mon, Mutex* fl_lock,
-                  int max_completed_queue = 0) {
+                  int max_completed_queue = 0,
+                  PtrQueueSet *fl_owner = NULL) {
     _max_completed_queue = max_completed_queue;
     assert(cbl_mon != NULL && fl_lock != NULL, "Init order issue?");
-    _cbl_mon = cbl_mon; _fl_lock = fl_lock;
+    _cbl_mon = cbl_mon;
+    _fl_lock = fl_lock;
+    _fl_owner = (fl_owner != NULL) ? fl_owner : this;
   }
 
   // Return an empty oop array of size _sz (required to be non-zero).
@@ -228,4 +234,7 @@
   void reduce_free_list();
 
   size_t completed_buffers_num() { return _n_completed_buffers; }
+
+  void merge_bufferlists(PtrQueueSet* src);
+  void merge_freelists(PtrQueueSet* src);
 };
--- a/src/share/vm/gc_implementation/g1/sparsePRT.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/sparsePRT.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -504,6 +504,7 @@
   // Make sure that the current and next tables agree.  (Another mechanism
   // takes care of deleting now-unused tables.)
   _cur = _next;
+  set_expanded(false);
 }
 
 void SparsePRT::expand() {
--- a/src/share/vm/gc_implementation/g1/sparsePRT.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/sparsePRT.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -274,7 +274,7 @@
 
   // Clean up all tables on the expanded list.  Called single threaded.
   static void cleanup_all();
-  RSHashTable* next() const { return _next; }
+  RSHashTable* cur() const { return _cur; }
 
 
   void init_iterator(SparsePRTIter* sprt_iter);
@@ -300,7 +300,7 @@
   {}
 
   void init(const SparsePRT* sprt) {
-    RSHashTableIter::init(sprt->next());
+    RSHashTableIter::init(sprt->cur());
   }
   bool has_next(size_t& card_index) {
     return RSHashTableIter::has_next(card_index);
--- a/src/share/vm/gc_implementation/g1/survRateGroup.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/survRateGroup.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/g1/survRateGroup.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/g1/survRateGroup.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2004-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2004-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/includeDB_gc_g1	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/includeDB_gc_g1	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2004-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2004-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/includeDB_gc_parallelScavenge	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/includeDB_gc_parallelScavenge	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/includeDB_gc_shared	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/includeDB_gc_shared	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parNew/parGCAllocBuffer.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parNew/parGCAllocBuffer.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/psMarkSweep.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psMarkSweep.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -125,6 +125,8 @@
     perm_gen->verify_object_start_array();
   }
 
+  heap->pre_full_gc_dump();
+
   // Filled in below to track the state of the young gen after the collection.
   bool eden_empty;
   bool survivors_empty;
@@ -363,6 +365,8 @@
     Universe::print_heap_after_gc();
   }
 
+  heap->post_full_gc_dump();
+
 #ifdef TRACESPINNING
   ParallelTaskTerminator::print_termination_counts();
 #endif
--- a/src/share/vm/gc_implementation/parallelScavenge/psMarkSweepDecorator.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psMarkSweepDecorator.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/psOldGen.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psOldGen.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1982,6 +1982,8 @@
     heap->record_gen_tops_before_GC();
   }
 
+  heap->pre_full_gc_dump();
+
   _print_phases = PrintGCDetails && PrintParallelOldGCPhaseTimes;
 
   // Make sure data structures are sane, make the heap parsable, and do other
@@ -2204,6 +2206,8 @@
     gc_task_manager()->print_task_time_stamps();
   }
 
+  heap->post_full_gc_dump();
+
 #ifdef TRACESPINNING
   ParallelTaskTerminator::print_termination_counts();
 #endif
--- a/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/psScavenge.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psScavenge.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/parallelScavenge/psYoungGen.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/parallelScavenge/psYoungGen.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/shared/ageTable.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/ageTable.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,7 +22,7 @@
  *
  */
 
-/* Copyright 1992 Sun Microsystems, Inc. and Stanford University.
+/* Copyright 1992-2009 Sun Microsystems, Inc. and Stanford University.
    See the LICENSE file for license information. */
 
 # include "incls/_precompiled.incl"
--- a/src/share/vm/gc_implementation/shared/ageTable.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/ageTable.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2003 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,7 +22,7 @@
  *
  */
 
-/* Copyright 1992 Sun Microsystems, Inc. and Stanford University.
+/* Copyright 1992-2009 Sun Microsystems, Inc. and Stanford University.
    See the LICENSE file for license information. */
 
 // Age table for adaptive feedback-mediated tenuring (scavenging)
--- a/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,6 +1,6 @@
 
 /*
- * Copyright 2006-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2006-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/shared/mutableNUMASpace.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/mutableNUMASpace.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2006-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2006-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/shared/mutableSpace.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/mutableSpace.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/shared/mutableSpace.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/mutableSpace.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/gc_implementation/shared/vmGCOperations.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/vmGCOperations.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -121,7 +121,7 @@
     // make the heap parsable (no need to retire TLABs)
     ch->ensure_parsability(false);
   }
-  HeapInspection::heap_inspection(_out);
+  HeapInspection::heap_inspection(_out, _need_prologue /* need_prologue */);
 }
 
 
--- a/src/share/vm/gc_implementation/shared/vmGCOperations.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_implementation/shared/vmGCOperations.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -112,13 +112,16 @@
  private:
   outputStream* _out;
   bool _full_gc;
+  bool _need_prologue;
  public:
-  VM_GC_HeapInspection(outputStream* out, bool request_full_gc) :
+  VM_GC_HeapInspection(outputStream* out, bool request_full_gc,
+                       bool need_prologue) :
     VM_GC_Operation(0 /* total collections,      dummy, ignored */,
                     0 /* total full collections, dummy, ignored */,
                     request_full_gc) {
     _out = out;
     _full_gc = request_full_gc;
+    _need_prologue = need_prologue;
   }
 
   ~VM_GC_HeapInspection() {}
--- a/src/share/vm/gc_interface/collectedHeap.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_interface/collectedHeap.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -294,3 +294,29 @@
     ThreadLocalAllocBuffer::resize_all_tlabs();
   }
 }
+
+void CollectedHeap::pre_full_gc_dump() {
+  if (HeapDumpBeforeFullGC) {
+    TraceTime tt("Heap Dump: ", PrintGCDetails, false, gclog_or_tty);
+    // We are doing a "major" collection and a heap dump before
+    // major collection has been requested.
+    HeapDumper::dump_heap();
+  }
+  if (PrintClassHistogramBeforeFullGC) {
+    TraceTime tt("Class Histogram: ", PrintGCDetails, true, gclog_or_tty);
+    VM_GC_HeapInspection inspector(gclog_or_tty, false /* ! full gc */, false /* ! prologue */);
+    inspector.doit();
+  }
+}
+
+void CollectedHeap::post_full_gc_dump() {
+  if (HeapDumpAfterFullGC) {
+    TraceTime tt("Heap Dump", PrintGCDetails, false, gclog_or_tty);
+    HeapDumper::dump_heap();
+  }
+  if (PrintClassHistogramAfterFullGC) {
+    TraceTime tt("Class Histogram", PrintGCDetails, true, gclog_or_tty);
+    VM_GC_HeapInspection inspector(gclog_or_tty, false /* ! full gc */, false /* ! prologue */);
+    inspector.doit();
+  }
+}
--- a/src/share/vm/gc_interface/collectedHeap.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_interface/collectedHeap.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -514,6 +514,10 @@
   // Perform any cleanup actions necessary before allowing a verification.
   virtual void prepare_for_verify() = 0;
 
+  // Generate any dumps preceding or following a full gc
+  void pre_full_gc_dump();
+  void post_full_gc_dump();
+
   virtual void print() const = 0;
   virtual void print_on(outputStream* st) const = 0;
 
--- a/src/share/vm/gc_interface/collectedHeap.inline.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/gc_interface/collectedHeap.inline.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/includeDB_compiler2	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/includeDB_compiler2	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/includeDB_features	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/includeDB_features	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2007-2008 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2007-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/includeDB_gc	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/includeDB_gc	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 //
-// Copyright 2001-2005 Sun Microsystems, Inc.  All Rights Reserved.
+// Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -26,10 +26,12 @@
 
 collectedHeap.cpp                       collectedHeap.hpp
 collectedHeap.cpp                       collectedHeap.inline.hpp
+collectedHeap.cpp                       heapDumper.hpp
 collectedHeap.cpp                       init.hpp
 collectedHeap.cpp                       oop.inline.hpp
 collectedHeap.cpp                       systemDictionary.hpp
 collectedHeap.cpp                       thread_<os_family>.inline.hpp
+collectedHeap.cpp                       vmGCOperations.hpp
 
 collectedHeap.hpp                       allocation.hpp
 collectedHeap.hpp                       barrierSet.hpp
--- a/src/share/vm/interpreter/bytecodeInterpreter.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/interpreter/bytecodeInterpreter.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/bytecodeInterpreter.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/interpreter/bytecodeInterpreter.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/interpreter/rewriter.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/interpreter/rewriter.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/libadt/dict.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/libadt/dict.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2003 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/libadt/port.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/libadt/port.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2003 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/cardTableModRefBS.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/cardTableModRefBS.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -356,18 +356,62 @@
   inline_write_ref_field(field, newVal);
 }
 
+/*
+   Claimed and deferred bits are used together in G1 during the evacuation
+   pause. These bits can have the following state transitions:
+   1. The claimed bit can be put over any other card state. Except that
+      the "dirty -> dirty and claimed" transition is checked for in
+      G1 code and is not used.
+   2. Deferred bit can be set only if the previous state of the card
+      was either clean or claimed. mark_card_deferred() is wait-free.
+      We do not care if the operation is be successful because if
+      it does not it will only result in duplicate entry in the update
+      buffer because of the "cache-miss". So it's not worth spinning.
+ */
+
 
 bool CardTableModRefBS::claim_card(size_t card_index) {
   jbyte val = _byte_map[card_index];
-  if (val != claimed_card_val()) {
-    jbyte res = Atomic::cmpxchg((jbyte) claimed_card_val(), &_byte_map[card_index], val);
-    if (res == val)
+  assert(val != dirty_card_val(), "Shouldn't claim a dirty card");
+  while (val == clean_card_val() ||
+         (val & (clean_card_mask_val() | claimed_card_val())) != claimed_card_val()) {
+    jbyte new_val = val;
+    if (val == clean_card_val()) {
+      new_val = (jbyte)claimed_card_val();
+    } else {
+      new_val = val | (jbyte)claimed_card_val();
+    }
+    jbyte res = Atomic::cmpxchg(new_val, &_byte_map[card_index], val);
+    if (res == val) {
       return true;
-    else return false;
+    }
+    val = res;
   }
   return false;
 }
 
+bool CardTableModRefBS::mark_card_deferred(size_t card_index) {
+  jbyte val = _byte_map[card_index];
+  // It's already processed
+  if ((val & (clean_card_mask_val() | deferred_card_val())) == deferred_card_val()) {
+    return false;
+  }
+  // Cached bit can be installed either on a clean card or on a claimed card.
+  jbyte new_val = val;
+  if (val == clean_card_val()) {
+    new_val = (jbyte)deferred_card_val();
+  } else {
+    if (val & claimed_card_val()) {
+      new_val = val | (jbyte)deferred_card_val();
+    }
+  }
+  if (new_val != val) {
+    Atomic::cmpxchg(new_val, &_byte_map[card_index], val);
+  }
+  return true;
+}
+
+
 void CardTableModRefBS::non_clean_card_iterate(Space* sp,
                                                MemRegion mr,
                                                DirtyCardToOopClosure* dcto_cl,
--- a/src/share/vm/memory/cardTableModRefBS.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/cardTableModRefBS.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,11 +52,15 @@
 
   enum CardValues {
     clean_card                  = -1,
+    // The mask contains zeros in places for all other values.
+    clean_card_mask             = clean_card - 31,
+
     dirty_card                  =  0,
     precleaned_card             =  1,
-    claimed_card                =  3,
-    last_card                   =  4,
-    CT_MR_BS_last_reserved      = 10
+    claimed_card                =  2,
+    deferred_card               =  4,
+    last_card                   =  8,
+    CT_MR_BS_last_reserved      = 16
   };
 
   // dirty and precleaned are equivalent wrt younger_refs_iter.
@@ -254,9 +258,11 @@
   };
 
   static int clean_card_val()      { return clean_card; }
+  static int clean_card_mask_val() { return clean_card_mask; }
   static int dirty_card_val()      { return dirty_card; }
   static int claimed_card_val()    { return claimed_card; }
   static int precleaned_card_val() { return precleaned_card; }
+  static int deferred_card_val()   { return deferred_card; }
 
   // For RTTI simulation.
   bool is_a(BarrierSet::Name bsn) {
@@ -329,7 +335,8 @@
   }
 
   bool is_card_claimed(size_t card_index) {
-    return _byte_map[card_index] == claimed_card_val();
+    jbyte val = _byte_map[card_index];
+    return (val & (clean_card_mask_val() | claimed_card_val())) == claimed_card_val();
   }
 
   bool claim_card(size_t card_index);
@@ -338,6 +345,13 @@
     return _byte_map[card_index] == clean_card_val();
   }
 
+  bool is_card_deferred(size_t card_index) {
+    jbyte val = _byte_map[card_index];
+    return (val & (clean_card_mask_val() | deferred_card_val())) == deferred_card_val();
+  }
+
+  bool mark_card_deferred(size_t card_index);
+
   // Card marking array base (adjusted for heap low boundary)
   // This would be the 0th element of _byte_map, if the heap started at 0x0.
   // But since the heap starts at some higher address, this points to somewhere
@@ -434,6 +448,10 @@
     return byte_for(p) - _byte_map;
   }
 
+  const jbyte* byte_for_index(const size_t card_index) const {
+    return _byte_map + card_index;
+  }
+
   void verify();
   void verify_guard();
 
--- a/src/share/vm/memory/genCollectedHeap.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/genCollectedHeap.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -456,6 +456,9 @@
     int max_level_collected = starting_level;
     for (int i = starting_level; i <= max_level; i++) {
       if (_gens[i]->should_collect(full, size, is_tlab)) {
+        if (i == n_gens() - 1) {  // a major collection is to happen
+          pre_full_gc_dump();    // do any pre full gc dumps
+        }
         // Timer for individual generations. Last argument is false: no CR
         TraceTime t1(_gens[i]->short_name(), PrintGCDetails, false, gclog_or_tty);
         TraceCollectorStats tcs(_gens[i]->counters());
@@ -573,6 +576,10 @@
     // a whole heap collection.
     complete = complete || (max_level_collected == n_gens() - 1);
 
+    if (complete) { // We did a "major" collection
+      post_full_gc_dump();   // do any post full gc dumps
+    }
+
     if (PrintGCDetails) {
       print_heap_change(gch_prev_used);
 
--- a/src/share/vm/memory/genCollectedHeap.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/genCollectedHeap.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/generation.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/generation.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/generation.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/generation.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/heapInspection.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/heapInspection.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -233,7 +233,7 @@
   size_t missed_count() { return _missed_count; }
 };
 
-void HeapInspection::heap_inspection(outputStream* st) {
+void HeapInspection::heap_inspection(outputStream* st, bool need_prologue) {
   ResourceMark rm;
   HeapWord* ref;
 
@@ -244,7 +244,9 @@
     case CollectedHeap::GenCollectedHeap: {
       is_shared_heap = true;
       SharedHeap* sh = (SharedHeap*)heap;
-      sh->gc_prologue(false /* !full */); // get any necessary locks, etc.
+      if (need_prologue) {
+        sh->gc_prologue(false /* !full */); // get any necessary locks, etc.
+      }
       ref = sh->perm_gen()->used_region().start();
       break;
     }
@@ -290,7 +292,7 @@
   }
   st->flush();
 
-  if (is_shared_heap) {
+  if (need_prologue && is_shared_heap) {
     SharedHeap* sh = (SharedHeap*)heap;
     sh->gc_epilogue(false /* !full */); // release all acquired locks, etc.
   }
--- a/src/share/vm/memory/heapInspection.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/heapInspection.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -127,6 +127,6 @@
 
 class HeapInspection : public AllStatic {
  public:
-  static void heap_inspection(outputStream* st) KERNEL_RETURN;
+  static void heap_inspection(outputStream* st, bool need_prologue) KERNEL_RETURN;
   static void find_instances_at_safepoint(klassOop k, GrowableArray<oop>* result) KERNEL_RETURN;
 };
--- a/src/share/vm/memory/oopFactory.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/oopFactory.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/oopFactory.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/oopFactory.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/permGen.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/permGen.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/referenceProcessor.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/referenceProcessor.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/sharedHeap.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/sharedHeap.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/sharedHeap.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/sharedHeap.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/space.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/space.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/space.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/space.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/tenuredGeneration.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/tenuredGeneration.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/threadLocalAllocBuffer.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/threadLocalAllocBuffer.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/universe.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/universe.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/memory/universe.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/memory/universe.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/arrayOop.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/arrayOop.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/constMethodKlass.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/constMethodKlass.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/constMethodKlass.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/constMethodKlass.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/constMethodOop.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/constMethodOop.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/constantPoolKlass.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/constantPoolKlass.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/constantPoolKlass.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/constantPoolKlass.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/constantPoolOop.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/constantPoolOop.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/constantPoolOop.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/constantPoolOop.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/cpCacheKlass.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/cpCacheKlass.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/cpCacheKlass.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/cpCacheKlass.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/cpCacheOop.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/cpCacheOop.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/klass.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/klass.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/methodOop.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/methodOop.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/methodOop.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/methodOop.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/oop.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/oop.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/oop.inline.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/oop.inline.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/oopsHierarchy.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/oopsHierarchy.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/typeArrayKlass.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/typeArrayKlass.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/oops/typeArrayKlass.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/oops/typeArrayKlass.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/block.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/block.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/c2_globals.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/c2_globals.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/cfgnode.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/cfgnode.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/chaitin.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/chaitin.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/chaitin.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/chaitin.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/classes.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/classes.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -129,11 +129,13 @@
 macro(LShiftI)
 macro(LShiftL)
 macro(LoadB)
+macro(LoadUB)
 macro(LoadUS)
 macro(LoadD)
 macro(LoadD_unaligned)
 macro(LoadF)
 macro(LoadI)
+macro(LoadUI2L)
 macro(LoadKlass)
 macro(LoadNKlass)
 macro(LoadL)
--- a/src/share/vm/opto/compile.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/compile.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2005,8 +2005,10 @@
   case Op_StoreP:
   case Op_StoreN:
   case Op_LoadB:
+  case Op_LoadUB:
   case Op_LoadUS:
   case Op_LoadI:
+  case Op_LoadUI2L:
   case Op_LoadKlass:
   case Op_LoadNKlass:
   case Op_LoadL:
--- a/src/share/vm/opto/gcm.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/gcm.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1901,7 +1901,8 @@
   for (int i = 0; i < _members.length(); i++) {
     CFGElement* s = _members.at(i);
     float block_freq = s->_freq * loop_freq;
-    if (block_freq < MIN_BLOCK_FREQUENCY) block_freq = MIN_BLOCK_FREQUENCY;
+    if (g_isnan(block_freq) || block_freq < MIN_BLOCK_FREQUENCY)
+      block_freq = MIN_BLOCK_FREQUENCY;
     s->_freq = block_freq;
   }
   CFGLoop* ch = _child;
--- a/src/share/vm/opto/graphKit.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/graphKit.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/graphKit.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/graphKit.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/idealGraphPrinter.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/idealGraphPrinter.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2007-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2007-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/ifg.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/ifg.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/lcm.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/lcm.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/live.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/live.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/loopnode.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/loopnode.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/macro.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/macro.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/matcher.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/matcher.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/memnode.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/memnode.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -778,7 +778,7 @@
            adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
          "use LoadRangeNode instead");
   switch (bt) {
-  case T_BOOLEAN:
+  case T_BOOLEAN: return new (C, 3) LoadUBNode(ctl, mem, adr, adr_type, rt->is_int()    );
   case T_BYTE:    return new (C, 3) LoadBNode (ctl, mem, adr, adr_type, rt->is_int()    );
   case T_INT:     return new (C, 3) LoadINode (ctl, mem, adr, adr_type, rt->is_int()    );
   case T_CHAR:    return new (C, 3) LoadUSNode(ctl, mem, adr, adr_type, rt->is_int()    );
@@ -1616,6 +1616,22 @@
   return LoadNode::Ideal(phase, can_reshape);
 }
 
+//--------------------------LoadUBNode::Ideal-------------------------------------
+//
+//  If the previous store is to the same address as this load,
+//  and the value stored was larger than a byte, replace this load
+//  with the value stored truncated to a byte.  If no truncation is
+//  needed, the replacement is done in LoadNode::Identity().
+//
+Node* LoadUBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
+  Node* mem = in(MemNode::Memory);
+  Node* value = can_see_stored_value(mem, phase);
+  if (value && !phase->type(value)->higher_equal(_type))
+    return new (phase->C, 3) AndINode(value, phase->intcon(0xFF));
+  // Identity call will handle the case where truncation is not needed.
+  return LoadNode::Ideal(phase, can_reshape);
+}
+
 //--------------------------LoadUSNode::Ideal-------------------------------------
 //
 //  If the previous store is to the same address as this load,
--- a/src/share/vm/opto/memnode.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/memnode.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -207,6 +207,19 @@
   virtual BasicType memory_type() const { return T_BYTE; }
 };
 
+//------------------------------LoadUBNode-------------------------------------
+// Load a unsigned byte (8bits unsigned) from memory
+class LoadUBNode : public LoadNode {
+public:
+  LoadUBNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeInt* ti = TypeInt::UBYTE )
+    : LoadNode(c, mem, adr, at, ti) {}
+  virtual int Opcode() const;
+  virtual uint ideal_reg() const { return Op_RegI; }
+  virtual Node* Ideal(PhaseGVN *phase, bool can_reshape);
+  virtual int store_Opcode() const { return Op_StoreB; }
+  virtual BasicType memory_type() const { return T_BYTE; }
+};
+
 //------------------------------LoadUSNode-------------------------------------
 // Load an unsigned short/char (16bits unsigned) from memory
 class LoadUSNode : public LoadNode {
@@ -232,6 +245,18 @@
   virtual BasicType memory_type() const { return T_INT; }
 };
 
+//------------------------------LoadUI2LNode-----------------------------------
+// Load an unsigned integer into long from memory
+class LoadUI2LNode : public LoadNode {
+public:
+  LoadUI2LNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeLong* t = TypeLong::UINT)
+    : LoadNode(c, mem, adr, at, t) {}
+  virtual int Opcode() const;
+  virtual uint ideal_reg() const { return Op_RegL; }
+  virtual int store_Opcode() const { return Op_StoreL; }
+  virtual BasicType memory_type() const { return T_LONG; }
+};
+
 //------------------------------LoadRangeNode----------------------------------
 // Load an array length from the array
 class LoadRangeNode : public LoadINode {
--- a/src/share/vm/opto/mulnode.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/mulnode.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -486,20 +486,23 @@
     return new (phase->C, 3) AndINode(ldus, phase->intcon(mask&0xFFFF));
   }
 
-  // Masking sign bits off of a Byte?  Let the matcher use an unsigned load
-  if( lop == Op_LoadB &&
-      (!in(0) && load->in(0)) &&
-      (mask == 0x000000FF) ) {
-    // Associate this node with the LoadB, so the matcher can see them together.
-    // If we don't do this, it is common for the LoadB to have one control
-    // edge, and the store or call containing this AndI to have a different
-    // control edge.  This will cause Label_Root to group the AndI with
-    // the encoding store or call, so the matcher has no chance to match
-    // this AndI together with the LoadB.  Setting the control edge here
-    // prevents Label_Root from grouping the AndI with the store or call,
-    // if it has a control edge that is inconsistent with the LoadB.
-    set_req(0, load->in(0));
-    return this;
+  // Masking sign bits off of a Byte?  Do an unsigned byte load.
+  if (lop == Op_LoadB && mask == 0x000000FF) {
+    return new (phase->C, 3) LoadUBNode(load->in(MemNode::Control),
+                                        load->in(MemNode::Memory),
+                                        load->in(MemNode::Address),
+                                        load->adr_type());
+  }
+
+  // Masking sign bits off of a Byte plus additional lower bits?  Do
+  // an unsigned byte load plus an and.
+  if (lop == Op_LoadB && (mask & 0xFFFFFF00) == 0) {
+    Node* ldub = new (phase->C, 3) LoadUBNode(load->in(MemNode::Control),
+                                              load->in(MemNode::Memory),
+                                              load->in(MemNode::Address),
+                                              load->adr_type());
+    ldub = phase->transform(ldub);
+    return new (phase->C, 3) AndINode(ldub, phase->intcon(mask));
   }
 
   // Masking off sign bits?  Dont make them!
@@ -599,12 +602,21 @@
   if( !t2 || !t2->is_con() ) return MulNode::Ideal(phase, can_reshape);
   const jlong mask = t2->get_con();
 
-  Node *rsh = in(1);
-  uint rop = rsh->Opcode();
+  Node* in1 = in(1);
+  uint op = in1->Opcode();
+
+  // Masking sign bits off of an integer?  Do an unsigned integer to long load.
+  if (op == Op_ConvI2L && in1->in(1)->Opcode() == Op_LoadI && mask == 0x00000000FFFFFFFFL) {
+    Node* load = in1->in(1);
+    return new (phase->C, 3) LoadUI2LNode(load->in(MemNode::Control),
+                                          load->in(MemNode::Memory),
+                                          load->in(MemNode::Address),
+                                          load->adr_type());
+  }
 
   // Masking off sign bits?  Dont make them!
-  if( rop == Op_RShiftL ) {
-    const TypeInt *t12 = phase->type(rsh->in(2))->isa_int();
+  if (op == Op_RShiftL) {
+    const TypeInt *t12 = phase->type(in1->in(2))->isa_int();
     if( t12 && t12->is_con() ) { // Shift is by a constant
       int shift = t12->get_con();
       shift &= BitsPerJavaLong - 1;  // semantics of Java shifts
@@ -613,7 +625,7 @@
       // bits survive.  NO sign-extension bits survive the maskings.
       if( (sign_bits_mask & mask) == 0 ) {
         // Use zero-fill shift instead
-        Node *zshift = phase->transform(new (phase->C, 3) URShiftLNode(rsh->in(1),rsh->in(2)));
+        Node *zshift = phase->transform(new (phase->C, 3) URShiftLNode(in1->in(1), in1->in(2)));
         return new (phase->C, 3) AndLNode( zshift, in(2) );
       }
     }
--- a/src/share/vm/opto/reg_split.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/reg_split.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2000-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/superword.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/superword.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2007-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2007-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/opto/type.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/type.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -226,6 +226,7 @@
   TypeInt::CC_LE   = TypeInt::make(-1, 0, WidenMin);
   TypeInt::CC_GE   = TypeInt::make( 0, 1, WidenMin);  // == TypeInt::BOOL
   TypeInt::BYTE    = TypeInt::make(-128,127,     WidenMin); // Bytes
+  TypeInt::UBYTE   = TypeInt::make(0, 255,       WidenMin); // Unsigned Bytes
   TypeInt::CHAR    = TypeInt::make(0,65535,      WidenMin); // Java chars
   TypeInt::SHORT   = TypeInt::make(-32768,32767, WidenMin); // Java shorts
   TypeInt::POS     = TypeInt::make(0,max_jint,   WidenMin); // Non-neg values
@@ -1022,6 +1023,7 @@
 const TypeInt *TypeInt::CC_LE;  // [-1,0]
 const TypeInt *TypeInt::CC_GE;  // [0,1] == BOOL (!)
 const TypeInt *TypeInt::BYTE;   // Bytes, -128 to 127
+const TypeInt *TypeInt::UBYTE;  // Unsigned Bytes, 0 to 255
 const TypeInt *TypeInt::CHAR;   // Java chars, 0-65535
 const TypeInt *TypeInt::SHORT;  // Java shorts, -32768-32767
 const TypeInt *TypeInt::POS;    // Positive 32-bit integers or zero
--- a/src/share/vm/opto/type.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/type.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -415,6 +415,7 @@
   static const TypeInt *CC_LE;  // [-1,0]
   static const TypeInt *CC_GE;  // [0,1] == BOOL (!)
   static const TypeInt *BYTE;
+  static const TypeInt *UBYTE;
   static const TypeInt *CHAR;
   static const TypeInt *SHORT;
   static const TypeInt *POS;
--- a/src/share/vm/opto/vectornode.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/opto/vectornode.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2007-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2007-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/prims/jni.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/prims/jni.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/prims/jvm.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/prims/jvm.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/prims/jvmtiRedefineClasses.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/prims/jvmtiRedefineClasses.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/prims/jvmtiTagMap.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/prims/jvmtiTagMap.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/runtime/arguments.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/arguments.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/runtime/arguments.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/arguments.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/runtime/globals.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/globals.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -662,6 +662,12 @@
   product(ccstrlist, OnOutOfMemoryError, "",                                \
           "Run user-defined commands on first java.lang.OutOfMemoryError")  \
                                                                             \
+  manageable(bool, HeapDumpBeforeFullGC, false,                             \
+          "Dump heap to file before any major stop-world GC")               \
+                                                                            \
+  manageable(bool, HeapDumpAfterFullGC, false,                              \
+          "Dump heap to file after any major stop-world GC")                \
+                                                                            \
   manageable(bool, HeapDumpOnOutOfMemoryError, false,                       \
           "Dump heap to file when java.lang.OutOfMemoryError is thrown")    \
                                                                             \
@@ -1971,6 +1977,12 @@
   product(bool, PrintHeapAtSIGBREAK, true,                                  \
           "Print heap layout in response to SIGBREAK")                      \
                                                                             \
+  manageable(bool, PrintClassHistogramBeforeFullGC, false,                  \
+          "Print a class histogram before any major stop-world GC")         \
+                                                                            \
+  manageable(bool, PrintClassHistogramAfterFullGC, false,                   \
+          "Print a class histogram after any major stop-world GC")          \
+                                                                            \
   manageable(bool, PrintClassHistogram, false,                              \
           "Print a histogram of class instances")                           \
                                                                             \
--- a/src/share/vm/runtime/javaCalls.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/javaCalls.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/runtime/memprofiler.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/memprofiler.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/runtime/os.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/os.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -207,7 +207,8 @@
         VMThread::execute(&op1);
         Universe::print_heap_at_SIGBREAK();
         if (PrintClassHistogram) {
-          VM_GC_HeapInspection op1(gclog_or_tty, true /* force full GC before heap inspection */);
+          VM_GC_HeapInspection op1(gclog_or_tty, true /* force full GC before heap inspection */,
+                                   true /* need_prologue */);
           VMThread::execute(&op1);
         }
         if (JvmtiExport::should_post_data_dump()) {
--- a/src/share/vm/runtime/os.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/os.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/runtime/safepoint.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/safepoint.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/runtime/sharedRuntime.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/sharedRuntime.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/runtime/synchronizer.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/runtime/synchronizer.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/services/attachListener.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/services/attachListener.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -194,7 +194,7 @@
     }
     live_objects_only = strcmp(arg0, "-live") == 0;
   }
-  VM_GC_HeapInspection heapop(out, live_objects_only /* request gc */);
+  VM_GC_HeapInspection heapop(out, live_objects_only /* request full gc */, true /* need_prologue */);
   VMThread::execute(&heapop);
   return JNI_OK;
 }
--- a/src/share/vm/services/heapDumper.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/services/heapDumper.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -347,7 +347,6 @@
   INITIAL_CLASS_COUNT = 200
 };
 
-
 // Supports I/O operations on a dump file
 
 class DumpWriter : public StackObj {
@@ -1303,7 +1302,9 @@
 // The VM operation that performs the heap dump
 class VM_HeapDumper : public VM_GC_Operation {
  private:
-  DumpWriter* _writer;
+  static VM_HeapDumper* _global_dumper;
+  static DumpWriter*    _global_writer;
+  DumpWriter*           _local_writer;
   bool _gc_before_heap_dump;
   bool _is_segmented_dump;
   jlong _dump_start;
@@ -1311,8 +1312,20 @@
   ThreadStackTrace** _stack_traces;
   int _num_threads;
 
-  // accessors
-  DumpWriter* writer() const                    { return _writer; }
+  // accessors and setters
+  static VM_HeapDumper* dumper()         {  assert(_global_dumper != NULL, "Error"); return _global_dumper; }
+  static DumpWriter* writer()            {  assert(_global_writer != NULL, "Error"); return _global_writer; }
+  void set_global_dumper() {
+    assert(_global_dumper == NULL, "Error");
+    _global_dumper = this;
+  }
+  void set_global_writer() {
+    assert(_global_writer == NULL, "Error");
+    _global_writer = _local_writer;
+  }
+  void clear_global_dumper() { _global_dumper = NULL; }
+  void clear_global_writer() { _global_writer = NULL; }
+
   bool is_segmented_dump() const                { return _is_segmented_dump; }
   void set_segmented_dump()                     { _is_segmented_dump = true; }
   jlong dump_start() const                      { return _dump_start; }
@@ -1357,7 +1370,7 @@
     VM_GC_Operation(0 /* total collections,      dummy, ignored */,
                     0 /* total full collections, dummy, ignored */,
                     gc_before_heap_dump) {
-    _writer = writer;
+    _local_writer = writer;
     _gc_before_heap_dump = gc_before_heap_dump;
     _is_segmented_dump = false;
     _dump_start = (jlong)-1;
@@ -1381,6 +1394,9 @@
   void doit();
 };
 
+VM_HeapDumper* VM_HeapDumper::_global_dumper = NULL;
+DumpWriter*    VM_HeapDumper::_global_writer = NULL;
+
 bool VM_HeapDumper::skip_operation() const {
   return false;
 }
@@ -1479,31 +1495,28 @@
 void VM_HeapDumper::do_load_class(klassOop k) {
   static u4 class_serial_num = 0;
 
-  VM_HeapDumper* dumper = ((VM_HeapDumper*)VMThread::vm_operation());
-  DumpWriter* writer = dumper->writer();
-
   // len of HPROF_LOAD_CLASS record
   u4 remaining = 2*oopSize + 2*sizeof(u4);
 
   // write a HPROF_LOAD_CLASS for the class and each array class
   do {
-    DumperSupport::write_header(writer, HPROF_LOAD_CLASS, remaining);
+    DumperSupport::write_header(writer(), HPROF_LOAD_CLASS, remaining);
 
     // class serial number is just a number
-    writer->write_u4(++class_serial_num);
+    writer()->write_u4(++class_serial_num);
 
     // class ID
     Klass* klass = Klass::cast(k);
-    writer->write_classID(klass);
+    writer()->write_classID(klass);
 
     // add the klassOop and class serial number pair
-    dumper->add_class_serial_number(klass, class_serial_num);
+    dumper()->add_class_serial_number(klass, class_serial_num);
 
-    writer->write_u4(STACK_TRACE_ID);
+    writer()->write_u4(STACK_TRACE_ID);
 
     // class name ID
     symbolOop name = klass->name();
-    writer->write_objectID(name);
+    writer()->write_objectID(name);
 
     // write a LOAD_CLASS record for the array type (if it exists)
     k = klass->array_klass_or_null();
@@ -1512,17 +1525,13 @@
 
 // writes a HPROF_GC_CLASS_DUMP record for the given class
 void VM_HeapDumper::do_class_dump(klassOop k) {
-  VM_HeapDumper* dumper = ((VM_HeapDumper*)VMThread::vm_operation());
-  DumpWriter* writer = dumper->writer();
-  DumperSupport::dump_class_and_array_classes(writer, k);
+  DumperSupport::dump_class_and_array_classes(writer(), k);
 }
 
 // writes a HPROF_GC_CLASS_DUMP records for a given basic type
 // array (and each multi-dimensional array too)
 void VM_HeapDumper::do_basic_type_array_class_dump(klassOop k) {
-  VM_HeapDumper* dumper = ((VM_HeapDumper*)VMThread::vm_operation());
-  DumpWriter* writer = dumper->writer();
-  DumperSupport::dump_basic_type_array_class(writer, k);
+  DumperSupport::dump_basic_type_array_class(writer(), k);
 }
 
 // Walk the stack of the given thread.
@@ -1658,6 +1667,11 @@
     ch->ensure_parsability(false);
   }
 
+  // At this point we should be the only dumper active, so
+  // the following should be safe.
+  set_global_dumper();
+  set_global_writer();
+
   // Write the file header - use 1.0.2 for large heaps, otherwise 1.0.1
   size_t used = ch->used();
   const char* header;
@@ -1667,6 +1681,7 @@
   } else {
     header = "JAVA PROFILE 1.0.1";
   }
+
   // header is few bytes long - no chance to overflow int
   writer()->write_raw((void*)header, (int)strlen(header));
   writer()->write_u1(0); // terminator
@@ -1723,6 +1738,10 @@
   // fixes up the length of the dump record. In the case of a segmented
   // heap then the HPROF_HEAP_DUMP_END record is also written.
   end_of_dump();
+
+  // Now we clear the global variables, so that a future dumper might run.
+  clear_global_dumper();
+  clear_global_writer();
 }
 
 void VM_HeapDumper::dump_stack_traces() {
@@ -1790,7 +1809,12 @@
 
   // generate the dump
   VM_HeapDumper dumper(&writer, _gc_before_heap_dump);
-  VMThread::execute(&dumper);
+  if (Thread::current()->is_VM_thread()) {
+    assert(SafepointSynchronize::is_at_safepoint(), "Expected to be called at a safepoint");
+    dumper.doit();
+  } else {
+    VMThread::execute(&dumper);
+  }
 
   // close dump file and record any error that the writer may have encountered
   writer.close();
@@ -1845,49 +1869,68 @@
   }
 }
 
-
-// Called by error reporting
+// Called by error reporting by a single Java thread outside of a JVM safepoint,
+// or by heap dumping by the VM thread during a (GC) safepoint. Thus, these various
+// callers are strictly serialized and guaranteed not to interfere below. For more
+// general use, however, this method will need modification to prevent
+// inteference when updating the static variables base_path and dump_file_seq below.
 void HeapDumper::dump_heap() {
-  static char path[JVM_MAXPATHLEN];
+  static char base_path[JVM_MAXPATHLEN] = {'\0'};
+  static uint dump_file_seq = 0;
+  char   my_path[JVM_MAXPATHLEN] = {'\0'};
 
   // The dump file defaults to java_pid<pid>.hprof in the current working
   // directory. HeapDumpPath=<file> can be used to specify an alternative
   // dump file name or a directory where dump file is created.
-  bool use_default_filename = true;
-  if (HeapDumpPath == NULL || HeapDumpPath[0] == '\0') {
-    path[0] = '\0'; // HeapDumpPath=<file> not specified
-  } else {
-    assert(strlen(HeapDumpPath) < sizeof(path), "HeapDumpPath too long");
-    strcpy(path, HeapDumpPath);
-    // check if the path is a directory (must exist)
-    DIR* dir = os::opendir(path);
-    if (dir == NULL) {
-      use_default_filename = false;
+  if (dump_file_seq == 0) { // first time in, we initialize base_path
+    bool use_default_filename = true;
+    if (HeapDumpPath == NULL || HeapDumpPath[0] == '\0') {
+      // HeapDumpPath=<file> not specified
     } else {
-      // HeapDumpPath specified a directory. We append a file separator
-      // (if needed).
-      os::closedir(dir);
-      size_t fs_len = strlen(os::file_separator());
-      if (strlen(path) >= fs_len) {
-        char* end = path;
-        end += (strlen(path) - fs_len);
-        if (strcmp(end, os::file_separator()) != 0) {
-          assert(strlen(path) + strlen(os::file_separator()) < sizeof(path),
-            "HeapDumpPath too long");
-          strcat(path, os::file_separator());
+      assert(strlen(HeapDumpPath) < sizeof(base_path), "HeapDumpPath too long");
+      strcpy(base_path, HeapDumpPath);
+      // check if the path is a directory (must exist)
+      DIR* dir = os::opendir(base_path);
+      if (dir == NULL) {
+        use_default_filename = false;
+      } else {
+        // HeapDumpPath specified a directory. We append a file separator
+        // (if needed).
+        os::closedir(dir);
+        size_t fs_len = strlen(os::file_separator());
+        if (strlen(base_path) >= fs_len) {
+          char* end = base_path;
+          end += (strlen(base_path) - fs_len);
+          if (strcmp(end, os::file_separator()) != 0) {
+            assert(strlen(base_path) + strlen(os::file_separator()) < sizeof(base_path),
+              "HeapDumpPath too long");
+            strcat(base_path, os::file_separator());
+          }
         }
       }
     }
+    // If HeapDumpPath wasn't a file name then we append the default name
+    if (use_default_filename) {
+      char fn[32];
+      sprintf(fn, "java_pid%d", os::current_process_id());
+      assert(strlen(base_path) + strlen(fn) < sizeof(base_path), "HeapDumpPath too long");
+      strcat(base_path, fn);
+    }
+    assert(strlen(base_path) < sizeof(my_path), "Buffer too small");
+    strcpy(my_path, base_path);
+  } else {
+    // Append a sequence number id for dumps following the first
+    char fn[33];
+    sprintf(fn, ".%d", dump_file_seq);
+    assert(strlen(base_path) + strlen(fn) < sizeof(my_path), "HeapDumpPath too long");
+    strcpy(my_path, base_path);
+    strcat(my_path, fn);
   }
-  // If HeapDumpPath wasn't a file name then we append the default name
-  if (use_default_filename) {
-    char fn[32];
-    sprintf(fn, "java_pid%d.hprof", os::current_process_id());
-    assert(strlen(path) + strlen(fn) < sizeof(path), "HeapDumpPath too long");
-    strcat(path, fn);
-  }
+  dump_file_seq++;   // increment seq number for next time we dump
+  assert(strlen(".hprof") + strlen(my_path) < sizeof(my_path), "HeapDumpPath too long");
+  strcat(my_path, ".hprof");
 
   HeapDumper dumper(false /* no GC before heap dump */,
                     true  /* send to tty */);
-  dumper.dump(path);
+  dumper.dump(my_path);
 }
--- a/src/share/vm/services/heapDumper.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/services/heapDumper.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -53,7 +53,7 @@
 
  public:
   HeapDumper(bool gc_before_heap_dump) :
-    _gc_before_heap_dump(gc_before_heap_dump), _error(NULL), _print_to_tty(false)  { }
+    _gc_before_heap_dump(gc_before_heap_dump), _error(NULL), _print_to_tty(false) { }
   HeapDumper(bool gc_before_heap_dump, bool print_to_tty) :
     _gc_before_heap_dump(gc_before_heap_dump), _error(NULL), _print_to_tty(print_to_tty) { }
 
--- a/src/share/vm/services/management.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/services/management.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2003-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/utilities/globalDefinitions_gcc.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/utilities/globalDefinitions_gcc.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1998-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1998-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/utilities/globalDefinitions_sparcWorks.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/utilities/globalDefinitions_sparcWorks.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/utilities/ostream.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/utilities/ostream.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/utilities/taskqueue.cpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/utilities/taskqueue.cpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/utilities/taskqueue.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/utilities/taskqueue.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/src/share/vm/utilities/workgroup.hpp	Sun Mar 15 18:11:46 2009 -0700
+++ b/src/share/vm/utilities/workgroup.hpp	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/test/Makefile	Sun Mar 15 18:11:46 2009 -0700
+++ b/test/Makefile	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 #
-# Copyright 1995-2008 Sun Microsystems, Inc.  All Rights Reserved.
+# Copyright 1995-2009 Sun Microsystems, Inc.  All Rights Reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
--- a/test/compiler/6757316/Test6757316.java	Sun Mar 15 18:11:46 2009 -0700
+++ b/test/compiler/6757316/Test6757316.java	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2008-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/test/compiler/6758234/Test6758234.java	Sun Mar 15 18:11:46 2009 -0700
+++ b/test/compiler/6758234/Test6758234.java	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2008-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/test/compiler/6775880/Test.java	Sun Mar 15 18:11:46 2009 -0700
+++ b/test/compiler/6775880/Test.java	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2008-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- a/test/compiler/6778657/Test.java	Sun Mar 15 18:11:46 2009 -0700
+++ b/test/compiler/6778657/Test.java	Mon Mar 16 08:50:53 2009 -0400
@@ -1,5 +1,5 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2008-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/compiler/6797305/Test6797305.java	Mon Mar 16 08:50:53 2009 -0400
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/**
+ * @test
+ * @bug 6797305
+ * @summary Add LoadUB and LoadUI opcode class
+ *
+ * @run main/othervm -Xcomp -XX:CompileOnly=Test6797305.loadB,Test6797305.loadB2L,Test6797305.loadUB,Test6797305.loadUBmask,Test6797305.loadUB2L,Test6797305.loadS,Test6797305.loadS2L,Test6797305.loadUS,Test6797305.loadUSmask,Test6797305.loadUS2L,Test6797305.loadI,Test6797305.loadI2L,Test6797305.loadUI2L,Test6797305.loadL Test6797305
+ */
+
+public class Test6797305 {
+    static final byte[]  ba = new byte[]  { -1 };
+    static final short[] sa = new short[] { -1 };
+    static final int[]   ia = new int[]   { -1 };
+    static final long[]  la = new long[]  { -1 };
+
+    public static void main(String[] args)
+    {
+        long b = loadB(ba);
+        if (b != -1)
+            throw new InternalError("loadB failed: " + b + " != " + -1);
+
+        long b2l = loadB2L(ba);
+        if (b2l != -1L)
+            throw new InternalError("loadB2L failed: " + b2l + " != " + -1L);
+
+        int ub = loadUB(ba);
+        if (ub != 0xFF)
+            throw new InternalError("loadUB failed: " + ub + " != " + 0xFF);
+
+        int ubmask = loadUBmask(ba);
+        if (ubmask != 0xFE)
+            throw new InternalError("loadUBmask failed: " + ubmask + " != " + 0xFE);
+
+        long ub2l = loadUB2L(ba);
+        if (ub2l != 0xFFL)
+            throw new InternalError("loadUB2L failed: " + ub2l + " != " + 0xFFL);
+
+        int s = loadS(sa);
+        if (s != -1)
+            throw new InternalError("loadS failed: " + s + " != " + -1);
+
+        long s2l = loadS2L(sa);
+        if (s2l != -1L)
+            throw new InternalError("loadS2L failed: " + s2l + " != " + -1L);
+
+        int us = loadUS(sa);
+        if (us != 0xFFFF)
+            throw new InternalError("loadUS failed: " + us + " != " + 0xFFFF);
+
+        int usmask = loadUSmask(sa);
+        if (usmask != 0xFFFE)
+            throw new InternalError("loadUBmask failed: " + ubmask + " != " + 0xFFFE);
+
+        long us2l = loadUS2L(sa);
+        if (us2l != 0xFFFFL)
+            throw new InternalError("loadUS2L failed: " + us2l + " != " + 0xFFFFL);
+
+        int i = loadI(ia);
+        if (i != -1)
+            throw new InternalError("loadI failed: " + i + " != " + -1);
+
+        long i2l = loadI2L(ia);
+        if (i2l != -1L)
+            throw new InternalError("loadI2L failed: " + i2l + " != " + -1L);
+
+        long ui2l = loadUI2L(ia);
+        if (ui2l != 0xFFFFFFFFL)
+            throw new InternalError("loadUI2L failed: " + ui2l + " != " + 0xFFFFFFFFL);
+
+        long l = loadL(la);
+        if (l != -1L)
+            throw new InternalError("loadL failed: " + l + " != " + -1L);
+    }
+
+    static int  loadB     (byte[] ba)  { return ba[0];               }
+    static long loadB2L   (byte[] ba)  { return ba[0];               }
+    static int  loadUB    (byte[] ba)  { return ba[0] & 0xFF;        }
+    static int  loadUBmask(byte[] ba)  { return ba[0] & 0xFE;        }
+    static long loadUB2L  (byte[] ba)  { return ba[0] & 0xFF;        }
+
+    static int  loadS     (short[] sa) { return sa[0];               }
+    static long loadS2L   (short[] sa) { return sa[0];               }
+    static int  loadUS    (short[] sa) { return sa[0] & 0xFFFF;      }
+    static int  loadUSmask(short[] sa) { return sa[0] & 0xFFFE;      }
+    static long loadUS2L  (short[] sa) { return sa[0] & 0xFFFF;      }
+
+    static int  loadI     (int[] ia)   { return ia[0];               }
+    static long loadI2L   (int[] ia)   { return ia[0];               }
+    static long loadUI2L  (int[] ia)   { return ia[0] & 0xFFFFFFFFL; }
+
+    static long loadL     (long[] la)  { return la[0];               }
+}