--- a/agent/src/share/classes/sun/jvm/hotspot/code/CodeCache.java	Fri Sep 25 04:39:54 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/code/CodeCache.java	Wed Sep 30 13:25:39 2009 -0700
@@ -33,6 +33,7 @@
 
 public class CodeCache {
   private static AddressField       heapField;
+  private static AddressField       scavengeRootNMethodsField;
   private static VirtualConstructor virtualConstructor;
 
   private CodeHeap heap;
@@ -49,6 +50,7 @@
     Type type = db.lookupType("CodeCache");
 
     heapField = type.getAddressField("_heap");
+    scavengeRootNMethodsField = type.getAddressField("_scavenge_root_nmethods");
 
     virtualConstructor = new VirtualConstructor(db);
     // Add mappings for all possible CodeBlob subclasses
@@ -67,6 +69,10 @@
     heap = (CodeHeap) VMObjectFactory.newObject(CodeHeap.class, heapField.getValue());
   }
 
+  public NMethod scavengeRootMethods() {
+    return (NMethod) VMObjectFactory.newObject(NMethod.class, scavengeRootNMethodsField.getValue());
+  }
+
   public boolean contains(Address p) {
     return getHeap().contains(p);
   }

--- a/agent/src/share/classes/sun/jvm/hotspot/code/NMethod.java	Fri Sep 25 04:39:54 2009 -0700
+++ b/agent/src/share/classes/sun/jvm/hotspot/code/NMethod.java	Wed Sep 30 13:25:39 2009 -0700
@@ -40,7 +40,10 @@
   /** != InvocationEntryBci if this nmethod is an on-stack replacement method */
   private static CIntegerField entryBCIField;
   /** To support simple linked-list chaining of nmethods */
-  private static AddressField  linkField;
+  private static AddressField  osrLinkField;
+  private static AddressField  scavengeRootLinkField;
+  private static CIntegerField scavengeRootStateField;
+
   /** Offsets for different nmethod parts */
   private static CIntegerField exceptionOffsetField;
   private static CIntegerField deoptOffsetField;
@@ -87,7 +90,10 @@
     zombieInstructionSizeField  = type.getCIntegerField("_zombie_instruction_size");
     methodField                 = type.getOopField("_method");
     entryBCIField               = type.getCIntegerField("_entry_bci");
-    linkField                   = type.getAddressField("_link");
+    osrLinkField                = type.getAddressField("_osr_link");
+    scavengeRootLinkField       = type.getAddressField("_scavenge_root_link");
+    scavengeRootStateField      = type.getCIntegerField("_scavenge_root_state");
+
     exceptionOffsetField        = type.getCIntegerField("_exception_offset");
     deoptOffsetField            = type.getCIntegerField("_deoptimize_offset");
     origPCOffsetField           = type.getCIntegerField("_orig_pc_offset");
@@ -219,10 +225,19 @@
     return getEntryBCI();
   }
 
-  public NMethod getLink() {
-    return (NMethod) VMObjectFactory.newObject(NMethod.class, linkField.getValue(addr));
+  public NMethod getOSRLink() {
+    return (NMethod) VMObjectFactory.newObject(NMethod.class, osrLinkField.getValue(addr));
   }
 
+  public NMethod getScavengeRootLink() {
+    return (NMethod) VMObjectFactory.newObject(NMethod.class, scavengeRootLinkField.getValue(addr));
+  }
+
+  public int getScavengeRootState() {
+    return (int) scavengeRootStateField.getValue(addr);
+  }
+
+
   /** Tells whether frames described by this nmethod can be
       deoptimized. Note: native wrappers cannot be deoptimized. */
   public boolean canBeDeoptimized() { return isJavaMethod(); }

--- a/src/cpu/sparc/vm/assembler_sparc.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/sparc/vm/assembler_sparc.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -4676,3 +4676,50 @@
     load_ptr_contents(base, G6_heapbase);
   }
 }
+
+// Compare char[] arrays aligned to 4 bytes.
+void MacroAssembler::char_arrays_equals(Register ary1, Register ary2,
+                                        Register limit, Register result,
+                                        Register chr1, Register chr2, Label& Ldone) {
+  Label Lvector, Lloop;
+  assert(chr1 == result, "should be the same");
+
+  // Note: limit contains number of bytes (2*char_elements) != 0.
+  andcc(limit, 0x2, chr1); // trailing character ?
+  br(Assembler::zero, false, Assembler::pt, Lvector);
+  delayed()->nop();
+
+  // compare the trailing char
+  sub(limit, sizeof(jchar), limit);
+  lduh(ary1, limit, chr1);
+  lduh(ary2, limit, chr2);
+  cmp(chr1, chr2);
+  br(Assembler::notEqual, true, Assembler::pt, Ldone);
+  delayed()->mov(G0, result);     // not equal
+
+  // only one char ?
+  br_on_reg_cond(rc_z, true, Assembler::pn, limit, Ldone);
+  delayed()->add(G0, 1, result); // zero-length arrays are equal
+
+  // word by word compare, dont't need alignment check
+  bind(Lvector);
+  // Shift ary1 and ary2 to the end of the arrays, negate limit
+  add(ary1, limit, ary1);
+  add(ary2, limit, ary2);
+  neg(limit, limit);
+
+  lduw(ary1, limit, chr1);
+  bind(Lloop);
+  lduw(ary2, limit, chr2);
+  cmp(chr1, chr2);
+  br(Assembler::notEqual, true, Assembler::pt, Ldone);
+  delayed()->mov(G0, result);     // not equal
+  inccc(limit, 2*sizeof(jchar));
+  // annul LDUW if branch is not taken to prevent access past end of array
+  br(Assembler::notZero, true, Assembler::pt, Lloop);
+  delayed()->lduw(ary1, limit, chr1); // hoisted
+
+  // Caller should set it:
+  // add(G0, 1, result); // equals
+}
+

--- a/src/cpu/sparc/vm/assembler_sparc.hpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/sparc/vm/assembler_sparc.hpp	Wed Sep 30 13:25:39 2009 -0700
@@ -2455,6 +2455,11 @@
   void inc_counter(address counter_addr, Register Rtmp1, Register Rtmp2);
   void inc_counter(int*    counter_addr, Register Rtmp1, Register Rtmp2);
 
+  // Compare char[] arrays aligned to 4 bytes.
+  void char_arrays_equals(Register ary1, Register ary2,
+                          Register limit, Register result,
+                          Register chr1, Register chr2, Label& Ldone);
+
 #undef VIRTUAL
 
 };

--- a/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -2171,7 +2171,7 @@
     // subtype which we can't check or src is the same array as dst
     // but not necessarily exactly of type default_type.
     Label known_ok, halt;
-    jobject2reg(op->expected_type()->encoding(), tmp);
+    jobject2reg(op->expected_type()->constant_encoding(), tmp);
     __ ld_ptr(dst, oopDesc::klass_offset_in_bytes(), tmp2);
     if (basic_type != T_OBJECT) {
       __ cmp(tmp, tmp2);
@@ -2429,7 +2429,7 @@
       assert(data->is_BitData(), "need BitData for checkcast");
       Register mdo      = k_RInfo;
       Register data_val = Rtmp1;
-      jobject2reg(md->encoding(), mdo);
+      jobject2reg(md->constant_encoding(), mdo);
 
       int mdo_offset_bias = 0;
       if (!Assembler::is_simm13(md->byte_offset_of_slot(data, DataLayout::header_offset()) + data->size_in_bytes())) {
@@ -2452,7 +2452,7 @@
     // patching may screw with our temporaries on sparc,
     // so let's do it before loading the class
     if (k->is_loaded()) {
-      jobject2reg(k->encoding(), k_RInfo);
+      jobject2reg(k->constant_encoding(), k_RInfo);
     } else {
       jobject2reg_with_patching(k_RInfo, op->info_for_patch());
     }
@@ -2513,7 +2513,7 @@
     // patching may screw with our temporaries on sparc,
     // so let's do it before loading the class
     if (k->is_loaded()) {
-      jobject2reg(k->encoding(), k_RInfo);
+      jobject2reg(k->constant_encoding(), k_RInfo);
     } else {
       jobject2reg_with_patching(k_RInfo, op->info_for_patch());
     }
@@ -2717,7 +2717,7 @@
   assert(op->tmp1()->is_single_cpu(), "tmp1 must be allocated");
   Register mdo  = op->mdo()->as_register();
   Register tmp1 = op->tmp1()->as_register();
-  jobject2reg(md->encoding(), mdo);
+  jobject2reg(md->constant_encoding(), mdo);
   int mdo_offset_bias = 0;
   if (!Assembler::is_simm13(md->byte_offset_of_slot(data, CounterData::count_offset()) +
                             data->size_in_bytes())) {
@@ -2774,7 +2774,7 @@
         if (receiver == NULL) {
           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)) -
                             mdo_offset_bias);
-          jobject2reg(known_klass->encoding(), tmp1);
+          jobject2reg(known_klass->constant_encoding(), tmp1);
           __ st_ptr(tmp1, recv_addr);
           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)) -
                             mdo_offset_bias);

--- a/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -896,7 +896,7 @@
   LIR_Opr len = length.result();
   BasicType elem_type = x->elt_type();
 
-  __ oop2reg(ciTypeArrayKlass::make(elem_type)->encoding(), klass_reg);
+  __ oop2reg(ciTypeArrayKlass::make(elem_type)->constant_encoding(), klass_reg);
 
   CodeStub* slow_path = new NewTypeArrayStub(klass_reg, len, reg, info);
   __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, elem_type, klass_reg, slow_path);

--- a/src/cpu/sparc/vm/sparc.ad	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/sparc/vm/sparc.ad	Wed Sep 30 13:25:39 2009 -0700
@@ -2838,63 +2838,41 @@
   %}
 
 
-  enc_class enc_String_Compare(o0RegP str1, o1RegP str2, g3RegP tmp1, g4RegP tmp2, notemp_iRegI result) %{
+  enc_class enc_String_Compare(o0RegP str1, o1RegP str2, g3RegI cnt1, g4RegI cnt2, notemp_iRegI result) %{
     Label Ldone, Lloop;
     MacroAssembler _masm(&cbuf);
 
     Register   str1_reg = reg_to_register_object($str1$$reg);
     Register   str2_reg = reg_to_register_object($str2$$reg);
-    Register   tmp1_reg = reg_to_register_object($tmp1$$reg);
-    Register   tmp2_reg = reg_to_register_object($tmp2$$reg);
+    Register   cnt1_reg = reg_to_register_object($cnt1$$reg);
+    Register   cnt2_reg = reg_to_register_object($cnt2$$reg);
     Register result_reg = reg_to_register_object($result$$reg);
 
-    // Get the first character position in both strings
-    //         [8] char array, [12] offset, [16] count
-    int  value_offset = java_lang_String:: value_offset_in_bytes();
-    int offset_offset = java_lang_String::offset_offset_in_bytes();
-    int  count_offset = java_lang_String:: count_offset_in_bytes();
-
-    // load str1 (jchar*) base address into tmp1_reg
-    __ load_heap_oop(str1_reg, value_offset, tmp1_reg);
-    __ ld(str1_reg, offset_offset, result_reg);
-    __ add(tmp1_reg, arrayOopDesc::base_offset_in_bytes(T_CHAR), tmp1_reg);
-    __   ld(str1_reg, count_offset, str1_reg); // hoisted
-    __ sll(result_reg, exact_log2(sizeof(jchar)), result_reg);
-    __   load_heap_oop(str2_reg, value_offset, tmp2_reg); // hoisted
-    __ add(result_reg, tmp1_reg, tmp1_reg);
-
-    // load str2 (jchar*) base address into tmp2_reg
-    // __ ld_ptr(str2_reg, value_offset, tmp2_reg); // hoisted
-    __ ld(str2_reg, offset_offset, result_reg);
-    __ add(tmp2_reg, arrayOopDesc::base_offset_in_bytes(T_CHAR), tmp2_reg);
-    __   ld(str2_reg, count_offset, str2_reg); // hoisted
-    __ sll(result_reg, exact_log2(sizeof(jchar)), result_reg);
-    __   subcc(str1_reg, str2_reg, O7); // hoisted
-    __ add(result_reg, tmp2_reg, tmp2_reg);
+    assert(result_reg != str1_reg &&
+           result_reg != str2_reg &&
+           result_reg != cnt1_reg &&
+           result_reg != cnt2_reg ,
+           "need different registers");
 
     // Compute the minimum of the string lengths(str1_reg) and the
     // difference of the string lengths (stack)
 
-    // discard string base pointers, after loading up the lengths
-    // __ ld(str1_reg, count_offset, str1_reg); // hoisted
-    // __ ld(str2_reg, count_offset, str2_reg); // hoisted
-
     // See if the lengths are different, and calculate min in str1_reg.
     // Stash diff in O7 in case we need it for a tie-breaker.
     Label Lskip;
-    // __ subcc(str1_reg, str2_reg, O7); // hoisted
-    __ sll(str1_reg, exact_log2(sizeof(jchar)), str1_reg); // scale the limit
+    __ subcc(cnt1_reg, cnt2_reg, O7);
+    __ sll(cnt1_reg, exact_log2(sizeof(jchar)), cnt1_reg); // scale the limit
     __ br(Assembler::greater, true, Assembler::pt, Lskip);
-    // str2 is shorter, so use its count:
-    __ delayed()->sll(str2_reg, exact_log2(sizeof(jchar)), str1_reg); // scale the limit
+    // cnt2 is shorter, so use its count:
+    __ delayed()->sll(cnt2_reg, exact_log2(sizeof(jchar)), cnt1_reg); // scale the limit
     __ bind(Lskip);
 
-    // reallocate str1_reg, str2_reg, result_reg
+    // reallocate cnt1_reg, cnt2_reg, result_reg
     // Note:  limit_reg holds the string length pre-scaled by 2
-    Register limit_reg =   str1_reg;
-    Register  chr2_reg =   str2_reg;
+    Register limit_reg =   cnt1_reg;
+    Register  chr2_reg =   cnt2_reg;
     Register  chr1_reg = result_reg;
-    // tmp{12} are the base pointers
+    // str{12} are the base pointers
 
     // Is the minimum length zero?
     __ cmp(limit_reg, (int)(0 * sizeof(jchar))); // use cast to resolve overloading ambiguity
@@ -2902,8 +2880,8 @@
     __ delayed()->mov(O7, result_reg);  // result is difference in lengths
 
     // Load first characters
-    __ lduh(tmp1_reg, 0, chr1_reg);
-    __ lduh(tmp2_reg, 0, chr2_reg);
+    __ lduh(str1_reg, 0, chr1_reg);
+    __ lduh(str2_reg, 0, chr2_reg);
 
     // Compare first characters
     __ subcc(chr1_reg, chr2_reg, chr1_reg);
@@ -2915,7 +2893,7 @@
       // Check after comparing first character to see if strings are equivalent
       Label LSkip2;
       // Check if the strings start at same location
-      __ cmp(tmp1_reg, tmp2_reg);
+      __ cmp(str1_reg, str2_reg);
       __ brx(Assembler::notEqual, true, Assembler::pt, LSkip2);
       __ delayed()->nop();
 
@@ -2932,23 +2910,23 @@
     __ br(Assembler::equal, true, Assembler::pn, Ldone);
     __ delayed()->mov(O7, result_reg);  // result is difference in lengths
 
-    // Shift tmp1_reg and tmp2_reg to the end of the arrays, negate limit
-    __ add(tmp1_reg, limit_reg, tmp1_reg);
-    __ add(tmp2_reg, limit_reg, tmp2_reg);
+    // Shift str1_reg and str2_reg to the end of the arrays, negate limit
+    __ add(str1_reg, limit_reg, str1_reg);
+    __ add(str2_reg, limit_reg, str2_reg);
     __ neg(chr1_reg, limit_reg);  // limit = -(limit-2)
 
     // Compare the rest of the characters
-    __ lduh(tmp1_reg, limit_reg, chr1_reg);
+    __ lduh(str1_reg, limit_reg, chr1_reg);
     __ bind(Lloop);
-    // __ lduh(tmp1_reg, limit_reg, chr1_reg); // hoisted
-    __ lduh(tmp2_reg, limit_reg, chr2_reg);
+    // __ lduh(str1_reg, limit_reg, chr1_reg); // hoisted
+    __ lduh(str2_reg, limit_reg, chr2_reg);
     __ subcc(chr1_reg, chr2_reg, chr1_reg);
     __ br(Assembler::notZero, false, Assembler::pt, Ldone);
     assert(chr1_reg == result_reg, "result must be pre-placed");
     __ delayed()->inccc(limit_reg, sizeof(jchar));
     // annul LDUH if branch is not taken to prevent access past end of string
     __ br(Assembler::notZero, true, Assembler::pt, Lloop);
-    __ delayed()->lduh(tmp1_reg, limit_reg, chr1_reg); // hoisted
+    __ delayed()->lduh(str1_reg, limit_reg, chr1_reg); // hoisted
 
     // If strings are equal up to min length, return the length difference.
     __ mov(O7, result_reg);
@@ -2957,125 +2935,80 @@
     __ bind(Ldone);
   %}
 
-enc_class enc_String_Equals(o0RegP str1, o1RegP str2, g3RegP tmp1, g4RegP tmp2, notemp_iRegI result) %{
-    Label Lword, Lword_loop, Lpost_word, Lchar, Lchar_loop, Ldone;
+enc_class enc_String_Equals(o0RegP str1, o1RegP str2, g3RegI cnt, notemp_iRegI result) %{
+    Label Lword_loop, Lpost_word, Lchar, Lchar_loop, Ldone;
     MacroAssembler _masm(&cbuf);
 
     Register   str1_reg = reg_to_register_object($str1$$reg);
     Register   str2_reg = reg_to_register_object($str2$$reg);
-    Register   tmp1_reg = reg_to_register_object($tmp1$$reg);
-    Register   tmp2_reg = reg_to_register_object($tmp2$$reg);
+    Register    cnt_reg = reg_to_register_object($cnt$$reg);
+    Register   tmp1_reg = O7;
     Register result_reg = reg_to_register_object($result$$reg);
 
-    // Get the first character position in both strings
-    //         [8] char array, [12] offset, [16] count
-    int  value_offset = java_lang_String:: value_offset_in_bytes();
-    int offset_offset = java_lang_String::offset_offset_in_bytes();
-    int  count_offset = java_lang_String:: count_offset_in_bytes();
-
-    // load str1 (jchar*) base address into tmp1_reg
-    __ load_heap_oop(Address(str1_reg, value_offset), tmp1_reg);
-    __ ld(Address(str1_reg, offset_offset), result_reg);
-    __ add(tmp1_reg, arrayOopDesc::base_offset_in_bytes(T_CHAR), tmp1_reg);
-    __    ld(Address(str1_reg, count_offset), str1_reg); // hoisted
-    __ sll(result_reg, exact_log2(sizeof(jchar)), result_reg);
-    __    load_heap_oop(Address(str2_reg, value_offset), tmp2_reg); // hoisted
-    __ add(result_reg, tmp1_reg, tmp1_reg);
-
-    // load str2 (jchar*) base address into tmp2_reg
-    // __ ld_ptr(Address(str2_reg, value_offset), tmp2_reg); // hoisted
-    __ ld(Address(str2_reg, offset_offset), result_reg);
-    __ add(tmp2_reg, arrayOopDesc::base_offset_in_bytes(T_CHAR), tmp2_reg);
-    __    ld(Address(str2_reg, count_offset), str2_reg); // hoisted
-    __ sll(result_reg, exact_log2(sizeof(jchar)), result_reg);
-    __   cmp(str1_reg, str2_reg); // hoisted
-    __ add(result_reg, tmp2_reg, tmp2_reg);
-
-    __ sll(str1_reg, exact_log2(sizeof(jchar)), str1_reg);
-    __ br(Assembler::notEqual, true, Assembler::pt, Ldone);
-    __ delayed()->mov(G0, result_reg);    // not equal
-
-    __ br_zero(Assembler::equal, true, Assembler::pn, str1_reg, Ldone);
-    __ delayed()->add(G0, 1, result_reg); //equals
-
-    __ cmp(tmp1_reg, tmp2_reg); //same string ?
+    assert(result_reg != str1_reg &&
+           result_reg != str2_reg &&
+           result_reg !=  cnt_reg &&
+           result_reg != tmp1_reg ,
+           "need different registers");
+
+    __ cmp(str1_reg, str2_reg); //same char[] ?
     __ brx(Assembler::equal, true, Assembler::pn, Ldone);
     __ delayed()->add(G0, 1, result_reg);
 
+    __ br_on_reg_cond(Assembler::rc_z, true, Assembler::pn, cnt_reg, Ldone);
+    __ delayed()->add(G0, 1, result_reg); // count == 0
+
     //rename registers
-    Register limit_reg =   str1_reg;
-    Register  chr2_reg =   str2_reg;
+    Register limit_reg =    cnt_reg;
     Register  chr1_reg = result_reg;
-    // tmp{12} are the base pointers
+    Register  chr2_reg =   tmp1_reg;
 
     //check for alignment and position the pointers to the ends
-    __ or3(tmp1_reg, tmp2_reg, chr1_reg);
-    __ andcc(chr1_reg, 0x3, chr1_reg); // notZero means at least one not 4-byte aligned
-    __ br(Assembler::notZero, false, Assembler::pn, Lchar);
-    __ delayed()->nop();
-
-    __ bind(Lword);
-    __ and3(limit_reg, 0x2, O7); //remember the remainder (either 0 or 2)
-    __ andn(limit_reg, 0x3, limit_reg);
-    __ br_zero(Assembler::zero, false, Assembler::pn, limit_reg, Lpost_word);
-    __ delayed()->nop();
-
-    __ add(tmp1_reg, limit_reg, tmp1_reg);
-    __ add(tmp2_reg, limit_reg, tmp2_reg);
-    __ neg(limit_reg);
-
-    __ lduw(tmp1_reg, limit_reg, chr1_reg);
-    __ bind(Lword_loop);
-    __ lduw(tmp2_reg, limit_reg, chr2_reg);
-    __ cmp(chr1_reg, chr2_reg);
-    __ br(Assembler::notEqual, true, Assembler::pt, Ldone);
-    __ delayed()->mov(G0, result_reg);
-    __ inccc(limit_reg, 2*sizeof(jchar));
-    // annul LDUW if branch i  s not taken to prevent access past end of string
-    __ br(Assembler::notZero, true, Assembler::pt, Lword_loop); //annul on taken
-    __ delayed()->lduw(tmp1_reg, limit_reg, chr1_reg); // hoisted
-
-    __ bind(Lpost_word);
-    __ br_zero(Assembler::zero, true, Assembler::pt, O7, Ldone);
-    __ delayed()->add(G0, 1, result_reg);
-
-    __ lduh(tmp1_reg, 0, chr1_reg);
-    __ lduh(tmp2_reg, 0, chr2_reg);
-    __ cmp (chr1_reg, chr2_reg);
-    __ br(Assembler::notEqual, true, Assembler::pt, Ldone);
-    __ delayed()->mov(G0, result_reg);
+    __ or3(str1_reg, str2_reg, chr1_reg);
+    __ andcc(chr1_reg, 0x3, chr1_reg);
+    // notZero means at least one not 4-byte aligned.
+    // We could optimize the case when both arrays are not aligned
+    // but it is not frequent case and it requires additional checks.
+    __ br(Assembler::notZero, false, Assembler::pn, Lchar); // char by char compare
+    __ delayed()->sll(limit_reg, exact_log2(sizeof(jchar)), limit_reg); // set byte count
+
+    // Compare char[] arrays aligned to 4 bytes.
+    __ char_arrays_equals(str1_reg, str2_reg, limit_reg, result_reg,
+                          chr1_reg, chr2_reg, Ldone);
     __ ba(false,Ldone);
     __ delayed()->add(G0, 1, result_reg);
 
+    // char by char compare
     __ bind(Lchar);
-    __ add(tmp1_reg, limit_reg, tmp1_reg);
-    __ add(tmp2_reg, limit_reg, tmp2_reg);
+    __ add(str1_reg, limit_reg, str1_reg);
+    __ add(str2_reg, limit_reg, str2_reg);
     __ neg(limit_reg); //negate count
 
-    __ lduh(tmp1_reg, limit_reg, chr1_reg);
+    __ lduh(str1_reg, limit_reg, chr1_reg);
+    // Lchar_loop
     __ bind(Lchar_loop);
-    __ lduh(tmp2_reg, limit_reg, chr2_reg);
+    __ lduh(str2_reg, limit_reg, chr2_reg);
     __ cmp(chr1_reg, chr2_reg);
     __ br(Assembler::notEqual, true, Assembler::pt, Ldone);
     __ delayed()->mov(G0, result_reg); //not equal
     __ inccc(limit_reg, sizeof(jchar));
     // annul LDUH if branch is not taken to prevent access past end of string
-    __ br(Assembler::notZero, true, Assembler::pt, Lchar_loop); //annul on taken
-    __ delayed()->lduh(tmp1_reg, limit_reg, chr1_reg); // hoisted
+    __ br(Assembler::notZero, true, Assembler::pt, Lchar_loop);
+    __ delayed()->lduh(str1_reg, limit_reg, chr1_reg); // hoisted
 
     __ add(G0, 1, result_reg);  //equal
 
     __ bind(Ldone);
   %}
 
-enc_class enc_Array_Equals(o0RegP ary1, o1RegP ary2, g3RegP tmp1, g4RegP tmp2, notemp_iRegI result) %{
+enc_class enc_Array_Equals(o0RegP ary1, o1RegP ary2, g3RegP tmp1, notemp_iRegI result) %{
     Label Lvector, Ldone, Lloop;
     MacroAssembler _masm(&cbuf);
 
     Register   ary1_reg = reg_to_register_object($ary1$$reg);
     Register   ary2_reg = reg_to_register_object($ary2$$reg);
     Register   tmp1_reg = reg_to_register_object($tmp1$$reg);
-    Register   tmp2_reg = reg_to_register_object($tmp2$$reg);
+    Register   tmp2_reg = O7;
     Register result_reg = reg_to_register_object($result$$reg);
 
     int length_offset  = arrayOopDesc::length_offset_in_bytes();
@@ -3101,7 +3034,7 @@
     __ br(Assembler::notEqual, true, Assembler::pn, Ldone);
     __ delayed()->mov(G0, result_reg);     // not equal
 
-    __ br_zero(Assembler::zero, true, Assembler::pn, tmp1_reg, Ldone);
+    __ br_on_reg_cond(Assembler::rc_z, true, Assembler::pn, tmp1_reg, Ldone);
     __ delayed()->add(G0, 1, result_reg); // zero-length arrays are equal
 
     // load array addresses
@@ -3109,45 +3042,16 @@
     __ add(ary2_reg, base_offset, ary2_reg);
 
     // renaming registers
-    Register chr1_reg  =  tmp2_reg;   // for characters in ary1
-    Register chr2_reg  =  result_reg; // for characters in ary2
+    Register chr1_reg  =  result_reg; // for characters in ary1
+    Register chr2_reg  =  tmp2_reg;   // for characters in ary2
     Register limit_reg =  tmp1_reg;   // length
 
     // set byte count
     __ sll(limit_reg, exact_log2(sizeof(jchar)), limit_reg);
-    __ andcc(limit_reg, 0x2, chr1_reg); //trailing character ?
-    __ br(Assembler::zero, false, Assembler::pt, Lvector);
-    __ delayed()->nop();
-
-    //compare the trailing char
-    __ sub(limit_reg, sizeof(jchar), limit_reg);
-    __ lduh(ary1_reg, limit_reg, chr1_reg);
-    __ lduh(ary2_reg, limit_reg, chr2_reg);
-    __ cmp(chr1_reg, chr2_reg);
-    __ br(Assembler::notEqual, true, Assembler::pt, Ldone);
-    __ delayed()->mov(G0, result_reg);     // not equal
-
-    // only one char ?
-    __ br_zero(Assembler::zero, true, Assembler::pn, limit_reg, Ldone);
-    __ delayed()->add(G0, 1, result_reg); // zero-length arrays are equal
-
-    __ bind(Lvector);
-    // Shift ary1_reg and ary2_reg to the end of the arrays, negate limit
-    __ add(ary1_reg, limit_reg, ary1_reg);
-    __ add(ary2_reg, limit_reg, ary2_reg);
-    __ neg(limit_reg, limit_reg);
-
-    __ lduw(ary1_reg, limit_reg, chr1_reg);
-    __ bind(Lloop);
-    __ lduw(ary2_reg, limit_reg, chr2_reg);
-    __ cmp(chr1_reg, chr2_reg);
-    __ br(Assembler::notEqual, false, Assembler::pt, Ldone);
-    __ delayed()->mov(G0, result_reg);     // not equal
-    __ inccc(limit_reg, 2*sizeof(jchar));
-    // annul LDUW if branch is not taken to prevent access past end of string
-    __ br(Assembler::notZero, true, Assembler::pt, Lloop); //annul on taken
-    __ delayed()->lduw(ary1_reg, limit_reg, chr1_reg); // hoisted
-
+
+    // Compare char[] arrays aligned to 4 bytes.
+    __ char_arrays_equals(ary1_reg, ary2_reg, limit_reg, result_reg,
+                          chr1_reg, chr2_reg, Ldone);
     __ add(G0, 1, result_reg); // equals
 
     __ bind(Ldone);
@@ -9471,33 +9375,33 @@
   ins_pipe(long_memory_op);
 %}
 
-instruct string_compare(o0RegP str1, o1RegP str2, g3RegP tmp1, g4RegP tmp2, notemp_iRegI result,
-                        o7RegI tmp3, flagsReg ccr) %{
-  match(Set result (StrComp str1 str2));
-  effect(USE_KILL str1, USE_KILL str2, KILL tmp1, KILL tmp2, KILL ccr, KILL tmp3);
+instruct string_compare(o0RegP str1, o1RegP str2, g3RegI cnt1, g4RegI cnt2, notemp_iRegI result,
+                        o7RegI tmp, flagsReg ccr) %{
+  match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
+  effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL ccr, KILL tmp);
   ins_cost(300);
-  format %{ "String Compare $str1,$str2 -> $result" %}
-  ins_encode( enc_String_Compare(str1, str2, tmp1, tmp2, result) );
+  format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp" %}
+  ins_encode( enc_String_Compare(str1, str2, cnt1, cnt2, result) );
   ins_pipe(long_memory_op);
 %}
 
-instruct string_equals(o0RegP str1, o1RegP str2, g3RegP tmp1, g4RegP tmp2, notemp_iRegI result,
-                       o7RegI tmp3, flagsReg ccr) %{
-  match(Set result (StrEquals str1 str2));
-  effect(USE_KILL str1, USE_KILL str2, KILL tmp1, KILL tmp2, KILL ccr, KILL tmp3);
+instruct string_equals(o0RegP str1, o1RegP str2, g3RegI cnt, notemp_iRegI result,
+                       o7RegI tmp, flagsReg ccr) %{
+  match(Set result (StrEquals (Binary str1 str2) cnt));
+  effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp, KILL ccr);
   ins_cost(300);
-  format %{ "String Equals $str1,$str2 -> $result" %}
-  ins_encode( enc_String_Equals(str1, str2, tmp1, tmp2, result) );
+  format %{ "String Equals $str1,$str2,$cnt -> $result   // KILL $tmp" %}
+  ins_encode( enc_String_Equals(str1, str2, cnt, result) );
   ins_pipe(long_memory_op);
 %}
 
-instruct array_equals(o0RegP ary1, o1RegP ary2, g3RegP tmp1, g4RegP tmp2, notemp_iRegI result,
-                        flagsReg ccr) %{
+instruct array_equals(o0RegP ary1, o1RegP ary2, g3RegI tmp1, notemp_iRegI result,
+                      o7RegI tmp2, flagsReg ccr) %{
   match(Set result (AryEq ary1 ary2));
   effect(USE_KILL ary1, USE_KILL ary2, KILL tmp1, KILL tmp2, KILL ccr);
   ins_cost(300);
-  format %{ "Array Equals $ary1,$ary2 -> $result" %}
-  ins_encode( enc_Array_Equals(ary1, ary2, tmp1, tmp2, result));
+  format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1,$tmp2" %}
+  ins_encode( enc_Array_Equals(ary1, ary2, tmp1, result));
   ins_pipe(long_memory_op);
 %}
 

--- a/src/cpu/x86/vm/assembler_x86.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/x86/vm/assembler_x86.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -8404,6 +8404,319 @@
 }
 #endif // _LP64
 
+// IndexOf substring.
+void MacroAssembler::string_indexof(Register str1, Register str2,
+                                    Register cnt1, Register cnt2, Register result,
+                                    XMMRegister vec, Register tmp) {
+  assert(UseSSE42Intrinsics, "SSE4.2 is required");
+
+  Label RELOAD_SUBSTR, PREP_FOR_SCAN, SCAN_TO_SUBSTR,
+        SCAN_SUBSTR, RET_NOT_FOUND, CLEANUP;
+
+  push(str1); // string addr
+  push(str2); // substr addr
+  push(cnt2); // substr count
+  jmpb(PREP_FOR_SCAN);
+
+  // Substr count saved at sp
+  // Substr saved at sp+1*wordSize
+  // String saved at sp+2*wordSize
+
+  // Reload substr for rescan
+  bind(RELOAD_SUBSTR);
+  movl(cnt2, Address(rsp, 0));
+  movptr(str2, Address(rsp, wordSize));
+  // We came here after the beginninig of the substring was
+  // matched but the rest of it was not so we need to search
+  // again. Start from the next element after the previous match.
+  subptr(str1, result); // Restore counter
+  shrl(str1, 1);
+  addl(cnt1, str1);
+  lea(str1, Address(result, 2)); // Reload string
+
+  // Load substr
+  bind(PREP_FOR_SCAN);
+  movdqu(vec, Address(str2, 0));
+  addl(cnt1, 8);  // prime the loop
+  subptr(str1, 16);
+
+  // Scan string for substr in 16-byte vectors
+  bind(SCAN_TO_SUBSTR);
+  subl(cnt1, 8);
+  addptr(str1, 16);
+
+  // pcmpestri
+  //   inputs:
+  //     xmm - substring
+  //     rax - substring length (elements count)
+  //     mem - scaned string
+  //     rdx - string length (elements count)
+  //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
+  //   outputs:
+  //     rcx - matched index in string
+  assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
+
+  pcmpestri(vec, Address(str1, 0), 0x0d);
+  jcc(Assembler::above, SCAN_TO_SUBSTR);      // CF == 0 && ZF == 0
+  jccb(Assembler::aboveEqual, RET_NOT_FOUND); // CF == 0
+
+  // Fallthrough: found a potential substr
+
+  // Make sure string is still long enough
+  subl(cnt1, tmp);
+  cmpl(cnt1, cnt2);
+  jccb(Assembler::negative, RET_NOT_FOUND);
+  // Compute start addr of substr
+  lea(str1, Address(str1, tmp, Address::times_2));
+  movptr(result, str1); // save
+
+  // Compare potential substr
+  addl(cnt1, 8);     // prime the loop
+  addl(cnt2, 8);
+  subptr(str1, 16);
+  subptr(str2, 16);
+
+  // Scan 16-byte vectors of string and substr
+  bind(SCAN_SUBSTR);
+  subl(cnt1, 8);
+  subl(cnt2, 8);
+  addptr(str1, 16);
+  addptr(str2, 16);
+  movdqu(vec, Address(str2, 0));
+  pcmpestri(vec, Address(str1, 0), 0x0d);
+  jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
+  jcc(Assembler::positive, SCAN_SUBSTR);     // SF == 0
+
+  // Compute substr offset
+  subptr(result, Address(rsp, 2*wordSize));
+  shrl(result, 1); // index
+  jmpb(CLEANUP);
+
+  bind(RET_NOT_FOUND);
+  movl(result, -1);
+
+  bind(CLEANUP);
+  addptr(rsp, 3*wordSize);
+}
+
+// Compare strings.
+void MacroAssembler::string_compare(Register str1, Register str2,
+                                    Register cnt1, Register cnt2, Register result,
+                                    XMMRegister vec1, XMMRegister vec2) {
+  Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
+
+  // Compute the minimum of the string lengths and the
+  // difference of the string lengths (stack).
+  // Do the conditional move stuff
+  movl(result, cnt1);
+  subl(cnt1, cnt2);
+  push(cnt1);
+  if (VM_Version::supports_cmov()) {
+    cmovl(Assembler::lessEqual, cnt2, result);
+  } else {
+    Label GT_LABEL;
+    jccb(Assembler::greater, GT_LABEL);
+    movl(cnt2, result);
+    bind(GT_LABEL);
+  }
+
+  // Is the minimum length zero?
+  testl(cnt2, cnt2);
+  jcc(Assembler::zero, LENGTH_DIFF_LABEL);
+
+  // Load first characters
+  load_unsigned_short(result, Address(str1, 0));
+  load_unsigned_short(cnt1, Address(str2, 0));
+
+  // Compare first characters
+  subl(result, cnt1);
+  jcc(Assembler::notZero,  POP_LABEL);
+  decrementl(cnt2);
+  jcc(Assembler::zero, LENGTH_DIFF_LABEL);
+
+  {
+    // Check after comparing first character to see if strings are equivalent
+    Label LSkip2;
+    // Check if the strings start at same location
+    cmpptr(str1, str2);
+    jccb(Assembler::notEqual, LSkip2);
+
+    // Check if the length difference is zero (from stack)
+    cmpl(Address(rsp, 0), 0x0);
+    jcc(Assembler::equal,  LENGTH_DIFF_LABEL);
+
+    // Strings might not be equivalent
+    bind(LSkip2);
+  }
+
+  // Advance to next character
+  addptr(str1, 2);
+  addptr(str2, 2);
+
+  if (UseSSE42Intrinsics) {
+    // With SSE4.2, use double quad vector compare
+    Label COMPARE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
+    // Setup to compare 16-byte vectors
+    movl(cnt1, cnt2);
+    andl(cnt2, 0xfffffff8); // cnt2 holds the vector count
+    andl(cnt1, 0x00000007); // cnt1 holds the tail count
+    testl(cnt2, cnt2);
+    jccb(Assembler::zero, COMPARE_TAIL);
+
+    lea(str2, Address(str2, cnt2, Address::times_2));
+    lea(str1, Address(str1, cnt2, Address::times_2));
+    negptr(cnt2);
+
+    bind(COMPARE_VECTORS);
+    movdqu(vec1, Address(str1, cnt2, Address::times_2));
+    movdqu(vec2, Address(str2, cnt2, Address::times_2));
+    pxor(vec1, vec2);
+    ptest(vec1, vec1);
+    jccb(Assembler::notZero, VECTOR_NOT_EQUAL);
+    addptr(cnt2, 8);
+    jcc(Assembler::notZero, COMPARE_VECTORS);
+    jmpb(COMPARE_TAIL);
+
+    // Mismatched characters in the vectors
+    bind(VECTOR_NOT_EQUAL);
+    lea(str1, Address(str1, cnt2, Address::times_2));
+    lea(str2, Address(str2, cnt2, Address::times_2));
+    movl(cnt1, 8);
+
+    // Compare tail (< 8 chars), or rescan last vectors to
+    // find 1st mismatched characters
+    bind(COMPARE_TAIL);
+    testl(cnt1, cnt1);
+    jccb(Assembler::zero, LENGTH_DIFF_LABEL);
+    movl(cnt2, cnt1);
+    // Fallthru to tail compare
+  }
+
+  // Shift str2 and str1 to the end of the arrays, negate min
+  lea(str1, Address(str1, cnt2, Address::times_2, 0));
+  lea(str2, Address(str2, cnt2, Address::times_2, 0));
+  negptr(cnt2);
+
+    // Compare the rest of the characters
+  bind(WHILE_HEAD_LABEL);
+  load_unsigned_short(result, Address(str1, cnt2, Address::times_2, 0));
+  load_unsigned_short(cnt1, Address(str2, cnt2, Address::times_2, 0));
+  subl(result, cnt1);
+  jccb(Assembler::notZero, POP_LABEL);
+  increment(cnt2);
+  jcc(Assembler::notZero, WHILE_HEAD_LABEL);
+
+  // Strings are equal up to min length.  Return the length difference.
+  bind(LENGTH_DIFF_LABEL);
+  pop(result);
+  jmpb(DONE_LABEL);
+
+  // Discard the stored length difference
+  bind(POP_LABEL);
+  addptr(rsp, wordSize);
+
+  // That's it
+  bind(DONE_LABEL);
+}
+
+// Compare char[] arrays aligned to 4 bytes or substrings.
+void MacroAssembler::char_arrays_equals(bool is_array_equ, Register ary1, Register ary2,
+                                        Register limit, Register result, Register chr,
+                                        XMMRegister vec1, XMMRegister vec2) {
+  Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR;
+
+  int length_offset  = arrayOopDesc::length_offset_in_bytes();
+  int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
+
+  // Check the input args
+  cmpptr(ary1, ary2);
+  jcc(Assembler::equal, TRUE_LABEL);
+
+  if (is_array_equ) {
+    // Need additional checks for arrays_equals.
+    andptr(ary1, ary2);
+    jcc(Assembler::zero, FALSE_LABEL); // One pointer is NULL
+
+    // Check the lengths
+    movl(limit, Address(ary1, length_offset));
+    cmpl(limit, Address(ary2, length_offset));
+    jcc(Assembler::notEqual, FALSE_LABEL);
+  }
+
+  // count == 0
+  testl(limit, limit);
+  jcc(Assembler::zero, TRUE_LABEL);
+
+  if (is_array_equ) {
+    // Load array address
+    lea(ary1, Address(ary1, base_offset));
+    lea(ary2, Address(ary2, base_offset));
+  }
+
+  shll(limit, 1);      // byte count != 0
+  movl(result, limit); // copy
+
+  if (UseSSE42Intrinsics) {
+    // With SSE4.2, use double quad vector compare
+    Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
+    // Compare 16-byte vectors
+    andl(result, 0x0000000e);  //   tail count (in bytes)
+    andl(limit, 0xfffffff0);   // vector count (in bytes)
+    jccb(Assembler::zero, COMPARE_TAIL);
+
+    lea(ary1, Address(ary1, limit, Address::times_1));
+    lea(ary2, Address(ary2, limit, Address::times_1));
+    negptr(limit);
+
+    bind(COMPARE_WIDE_VECTORS);
+    movdqu(vec1, Address(ary1, limit, Address::times_1));
+    movdqu(vec2, Address(ary2, limit, Address::times_1));
+    pxor(vec1, vec2);
+    ptest(vec1, vec1);
+    jccb(Assembler::notZero, FALSE_LABEL);
+    addptr(limit, 16);
+    jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
+
+    bind(COMPARE_TAIL); // limit is zero
+    movl(limit, result);
+    // Fallthru to tail compare
+  }
+
+  // Compare 4-byte vectors
+  andl(limit, 0xfffffffc); // vector count (in bytes)
+  jccb(Assembler::zero, COMPARE_CHAR);
+
+  lea(ary1, Address(ary1, limit, Address::times_1));
+  lea(ary2, Address(ary2, limit, Address::times_1));
+  negptr(limit);
+
+  bind(COMPARE_VECTORS);
+  movl(chr, Address(ary1, limit, Address::times_1));
+  cmpl(chr, Address(ary2, limit, Address::times_1));
+  jccb(Assembler::notEqual, FALSE_LABEL);
+  addptr(limit, 4);
+  jcc(Assembler::notZero, COMPARE_VECTORS);
+
+  // Compare trailing char (final 2 bytes), if any
+  bind(COMPARE_CHAR);
+  testl(result, 0x2);   // tail  char
+  jccb(Assembler::zero, TRUE_LABEL);
+  load_unsigned_short(chr, Address(ary1, 0));
+  load_unsigned_short(limit, Address(ary2, 0));
+  cmpl(chr, limit);
+  jccb(Assembler::notEqual, FALSE_LABEL);
+
+  bind(TRUE_LABEL);
+  movl(result, 1);   // return true
+  jmpb(DONE);
+
+  bind(FALSE_LABEL);
+  xorl(result, result); // return false
+
+  // That's it
+  bind(DONE);
+}
+
 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
   switch (cond) {
     // Note some conditions are synonyms for others

--- a/src/cpu/x86/vm/assembler_x86.hpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/x86/vm/assembler_x86.hpp	Wed Sep 30 13:25:39 2009 -0700
@@ -2206,6 +2206,20 @@
   void movl2ptr(Register dst, Address src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); }
   void movl2ptr(Register dst, Register src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(if (dst != src) movl(dst, src)); }
 
+  // IndexOf strings.
+  void string_indexof(Register str1, Register str2,
+                      Register cnt1, Register cnt2, Register result,
+                      XMMRegister vec, Register tmp);
+
+  // Compare strings.
+  void string_compare(Register str1, Register str2,
+                      Register cnt1, Register cnt2, Register result,
+                      XMMRegister vec1, XMMRegister vec2);
+
+  // Compare char[] arrays.
+  void char_arrays_equals(bool is_array_equ, Register ary1, Register ary2,
+                          Register limit, Register result, Register chr,
+                          XMMRegister vec1, XMMRegister vec2);
 
 #undef VIRTUAL
 

--- a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -1638,7 +1638,7 @@
       jobject2reg_with_patching(k_RInfo, op->info_for_patch());
     } else {
 #ifdef _LP64
-      __ movoop(k_RInfo, k->encoding());
+      __ movoop(k_RInfo, k->constant_encoding());
 #else
       k_RInfo = noreg;
 #endif // _LP64
@@ -1661,7 +1661,7 @@
       assert(data != NULL,       "need data for checkcast");
       assert(data->is_BitData(), "need BitData for checkcast");
       Register mdo  = klass_RInfo;
-      __ movoop(mdo, md->encoding());
+      __ movoop(mdo, md->constant_encoding());
       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
       __ orl(data_addr, header_bits);
@@ -1679,7 +1679,7 @@
 #ifdef _LP64
         __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
 #else
-        __ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->encoding());
+        __ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
 #endif // _LP64
       } else {
         __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
@@ -1696,7 +1696,7 @@
 #ifdef _LP64
         __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
 #else
-        __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->encoding());
+        __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
 #endif // _LP64
         if (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes() != k->super_check_offset()) {
           __ jcc(Assembler::notEqual, *stub->entry());
@@ -1707,7 +1707,7 @@
 #ifdef _LP64
           __ cmpptr(klass_RInfo, k_RInfo);
 #else
-          __ cmpoop(klass_RInfo, k->encoding());
+          __ cmpoop(klass_RInfo, k->constant_encoding());
 #endif // _LP64
           __ jcc(Assembler::equal, done);
 
@@ -1715,7 +1715,7 @@
 #ifdef _LP64
           __ push(k_RInfo);
 #else
-          __ pushoop(k->encoding());
+          __ pushoop(k->constant_encoding());
 #endif // _LP64
           __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
           __ pop(klass_RInfo);
@@ -1763,7 +1763,7 @@
     if (!k->is_loaded()) {
       jobject2reg_with_patching(k_RInfo, op->info_for_patch());
     } else {
-      LP64_ONLY(__ movoop(k_RInfo, k->encoding()));
+      LP64_ONLY(__ movoop(k_RInfo, k->constant_encoding()));
     }
     assert(obj != k_RInfo, "must be different");
 
@@ -1774,7 +1774,7 @@
       // get object class
       // not a safepoint as obj null check happens earlier
       if (LP64_ONLY(false &&) k->is_loaded()) {
-        NOT_LP64(__ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->encoding()));
+        NOT_LP64(__ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding()));
         k_RInfo = noreg;
       } else {
         __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
@@ -1791,14 +1791,14 @@
 #ifndef _LP64
       if (k->is_loaded()) {
         // See if we get an immediate positive hit
-        __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->encoding());
+        __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
         __ jcc(Assembler::equal, one);
         if (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes() == k->super_check_offset()) {
           // check for self
-          __ cmpoop(klass_RInfo, k->encoding());
+          __ cmpoop(klass_RInfo, k->constant_encoding());
           __ jcc(Assembler::equal, one);
           __ push(klass_RInfo);
-          __ pushoop(k->encoding());
+          __ pushoop(k->constant_encoding());
           __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
           __ pop(klass_RInfo);
           __ pop(dst);
@@ -3112,7 +3112,7 @@
     // subtype which we can't check or src is the same array as dst
     // but not necessarily exactly of type default_type.
     Label known_ok, halt;
-    __ movoop(tmp, default_type->encoding());
+    __ movoop(tmp, default_type->constant_encoding());
     if (basic_type != T_OBJECT) {
       __ cmpptr(tmp, dst_klass_addr);
       __ jcc(Assembler::notEqual, halt);
@@ -3200,7 +3200,7 @@
   assert(data->is_CounterData(), "need CounterData for calls");
   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
   Register mdo  = op->mdo()->as_register();
-  __ movoop(mdo, md->encoding());
+  __ movoop(mdo, md->constant_encoding());
   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
   __ addl(counter_addr, DataLayout::counter_increment);
   Bytecodes::Code bc = method->java_code_at_bci(bci);
@@ -3240,7 +3240,7 @@
         ciKlass* receiver = vc_data->receiver(i);
         if (receiver == NULL) {
           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
-          __ movoop(recv_addr, known_klass->encoding());
+          __ movoop(recv_addr, known_klass->constant_encoding());
           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
           __ addl(data_addr, DataLayout::counter_increment);
           return;

--- a/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -994,7 +994,7 @@
   LIR_Opr len = length.result();
   BasicType elem_type = x->elt_type();
 
-  __ oop2reg(ciTypeArrayKlass::make(elem_type)->encoding(), klass_reg);
+  __ oop2reg(ciTypeArrayKlass::make(elem_type)->constant_encoding(), klass_reg);
 
   CodeStub* slow_path = new NewTypeArrayStub(klass_reg, len, reg, info);
   __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, elem_type, klass_reg, slow_path);

--- a/src/cpu/x86/vm/x86_32.ad	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/x86/vm/x86_32.ad	Wed Sep 30 13:25:39 2009 -0700
@@ -379,7 +379,7 @@
         int format) {
 #ifdef ASSERT
   if (rspec.reloc()->type() == relocInfo::oop_type && d32 != 0 && d32 != (int)Universe::non_oop_word()) {
-    assert(oop(d32)->is_oop() && oop(d32)->is_perm(), "cannot embed non-perm oops in code");
+    assert(oop(d32)->is_oop() && (ScavengeRootsInCode || !oop(d32)->is_scavengable()), "cannot embed scavengable oops in code");
   }
 #endif
   cbuf.relocate(cbuf.inst_mark(), rspec, format);
@@ -3701,458 +3701,6 @@
     }
   %}
 
-  enc_class enc_String_Compare(eDIRegP str1, eSIRegP str2, regXD tmp1, regXD tmp2,
-                        eAXRegI tmp3, eBXRegI tmp4, eCXRegI result) %{
-    Label ECX_GOOD_LABEL, LENGTH_DIFF_LABEL,
-          POP_LABEL, DONE_LABEL, CONT_LABEL,
-          WHILE_HEAD_LABEL;
-    MacroAssembler masm(&cbuf);
-
-    XMMRegister tmp1Reg   = as_XMMRegister($tmp1$$reg);
-    XMMRegister tmp2Reg   = as_XMMRegister($tmp2$$reg);
-
-    // Get the first character position in both strings
-    //         [8] char array, [12] offset, [16] count
-    int value_offset  = java_lang_String::value_offset_in_bytes();
-    int offset_offset = java_lang_String::offset_offset_in_bytes();
-    int count_offset  = java_lang_String::count_offset_in_bytes();
-    int base_offset   = arrayOopDesc::base_offset_in_bytes(T_CHAR);
-
-    masm.movptr(rax, Address(rsi, value_offset));
-    masm.movl(rcx, Address(rsi, offset_offset));
-    masm.lea(rax, Address(rax, rcx, Address::times_2, base_offset));
-    masm.movptr(rbx, Address(rdi, value_offset));
-    masm.movl(rcx, Address(rdi, offset_offset));
-    masm.lea(rbx, Address(rbx, rcx, Address::times_2, base_offset));
-
-    // Compute the minimum of the string lengths(rsi) and the
-    // difference of the string lengths (stack)
-
-    if (VM_Version::supports_cmov()) {
-      masm.movl(rdi, Address(rdi, count_offset));
-      masm.movl(rsi, Address(rsi, count_offset));
-      masm.movl(rcx, rdi);
-      masm.subl(rdi, rsi);
-      masm.push(rdi);
-      masm.cmovl(Assembler::lessEqual, rsi, rcx);
-    } else {
-      masm.movl(rdi, Address(rdi, count_offset));
-      masm.movl(rcx, Address(rsi, count_offset));
-      masm.movl(rsi, rdi);
-      masm.subl(rdi, rcx);
-      masm.push(rdi);
-      masm.jccb(Assembler::lessEqual, ECX_GOOD_LABEL);
-      masm.movl(rsi, rcx);
-      // rsi holds min, rcx is unused
-    }
-
-    // Is the minimum length zero?
-    masm.bind(ECX_GOOD_LABEL);
-    masm.testl(rsi, rsi);
-    masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL);
-
-    // Load first characters
-    masm.load_unsigned_short(rcx, Address(rbx, 0));
-    masm.load_unsigned_short(rdi, Address(rax, 0));
-
-    // Compare first characters
-    masm.subl(rcx, rdi);
-    masm.jcc(Assembler::notZero,  POP_LABEL);
-    masm.decrementl(rsi);
-    masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL);
-
-    {
-      // Check after comparing first character to see if strings are equivalent
-      Label LSkip2;
-      // Check if the strings start at same location
-      masm.cmpptr(rbx,rax);
-      masm.jccb(Assembler::notEqual, LSkip2);
-
-      // Check if the length difference is zero (from stack)
-      masm.cmpl(Address(rsp, 0), 0x0);
-      masm.jcc(Assembler::equal,  LENGTH_DIFF_LABEL);
-
-      // Strings might not be equivalent
-      masm.bind(LSkip2);
-    }
-
-   // Advance to next character
-    masm.addptr(rax, 2);
-    masm.addptr(rbx, 2);
-
-    if (UseSSE42Intrinsics) {
-      // With SSE4.2, use double quad vector compare
-      Label COMPARE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
-      // Setup to compare 16-byte vectors
-      masm.movl(rdi, rsi);
-      masm.andl(rsi, 0xfffffff8); // rsi holds the vector count
-      masm.andl(rdi, 0x00000007); // rdi holds the tail count
-      masm.testl(rsi, rsi);
-      masm.jccb(Assembler::zero, COMPARE_TAIL);
-
-      masm.lea(rax, Address(rax, rsi, Address::times_2));
-      masm.lea(rbx, Address(rbx, rsi, Address::times_2));
-      masm.negl(rsi);
-
-      masm.bind(COMPARE_VECTORS);
-      masm.movdqu(tmp1Reg, Address(rax, rsi, Address::times_2));
-      masm.movdqu(tmp2Reg, Address(rbx, rsi, Address::times_2));
-      masm.pxor(tmp1Reg, tmp2Reg);
-      masm.ptest(tmp1Reg, tmp1Reg);
-      masm.jccb(Assembler::notZero, VECTOR_NOT_EQUAL);
-      masm.addl(rsi, 8);
-      masm.jcc(Assembler::notZero, COMPARE_VECTORS);
-      masm.jmpb(COMPARE_TAIL);
-
-      // Mismatched characters in the vectors
-      masm.bind(VECTOR_NOT_EQUAL);
-      masm.lea(rax, Address(rax, rsi, Address::times_2));
-      masm.lea(rbx, Address(rbx, rsi, Address::times_2));
-      masm.movl(rdi, 8);
-
-      // Compare tail (< 8 chars), or rescan last vectors to
-      // find 1st mismatched characters
-      masm.bind(COMPARE_TAIL);
-      masm.testl(rdi, rdi);
-      masm.jccb(Assembler::zero, LENGTH_DIFF_LABEL);
-      masm.movl(rsi, rdi);
-      // Fallthru to tail compare
-    }
-
-    //Shift rax, and rbx, to the end of the arrays, negate min
-    masm.lea(rax, Address(rax, rsi, Address::times_2, 0));
-    masm.lea(rbx, Address(rbx, rsi, Address::times_2, 0));
-    masm.negl(rsi);
-
-    // Compare the rest of the characters
-    masm.bind(WHILE_HEAD_LABEL);
-    masm.load_unsigned_short(rcx, Address(rbx, rsi, Address::times_2, 0));
-    masm.load_unsigned_short(rdi, Address(rax, rsi, Address::times_2, 0));
-    masm.subl(rcx, rdi);
-    masm.jccb(Assembler::notZero, POP_LABEL);
-    masm.incrementl(rsi);
-    masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL);
-
-    // Strings are equal up to min length.  Return the length difference.
-    masm.bind(LENGTH_DIFF_LABEL);
-    masm.pop(rcx);
-    masm.jmpb(DONE_LABEL);
-
-    // Discard the stored length difference
-    masm.bind(POP_LABEL);
-    masm.addptr(rsp, 4);
-
-    // That's it
-    masm.bind(DONE_LABEL);
-  %}
-
- enc_class enc_String_Equals(eDIRegP str1, eSIRegP str2, regXD tmp1, regXD tmp2,
-                       eBXRegI tmp3, eCXRegI tmp4, eAXRegI result) %{
-    Label RET_TRUE, RET_FALSE, DONE, COMPARE_VECTORS, COMPARE_CHAR;
-    MacroAssembler masm(&cbuf);
-
-    XMMRegister tmp1Reg   = as_XMMRegister($tmp1$$reg);
-    XMMRegister tmp2Reg   = as_XMMRegister($tmp2$$reg);
-
-    int value_offset  = java_lang_String::value_offset_in_bytes();
-    int offset_offset = java_lang_String::offset_offset_in_bytes();
-    int count_offset  = java_lang_String::count_offset_in_bytes();
-    int base_offset   = arrayOopDesc::base_offset_in_bytes(T_CHAR);
-
-    // does source == target string?
-    masm.cmpptr(rdi, rsi);
-    masm.jcc(Assembler::equal, RET_TRUE);
-
-    // get and compare counts
-    masm.movl(rcx, Address(rdi, count_offset));
-    masm.movl(rax, Address(rsi, count_offset));
-    masm.cmpl(rcx, rax);
-    masm.jcc(Assembler::notEqual, RET_FALSE);
-    masm.testl(rax, rax);
-    masm.jcc(Assembler::zero, RET_TRUE);
-
-    // get source string offset and value
-    masm.movptr(rbx, Address(rsi, value_offset));
-    masm.movl(rax, Address(rsi, offset_offset));
-    masm.leal(rsi, Address(rbx, rax, Address::times_2, base_offset));
-
-    // get compare string offset and value
-    masm.movptr(rbx, Address(rdi, value_offset));
-    masm.movl(rax, Address(rdi, offset_offset));
-    masm.leal(rdi, Address(rbx, rax, Address::times_2, base_offset));
-
-    // Set byte count
-    masm.shll(rcx, 1);
-    masm.movl(rax, rcx);
-
-    if (UseSSE42Intrinsics) {
-      // With SSE4.2, use double quad vector compare
-      Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
-      // Compare 16-byte vectors
-      masm.andl(rcx, 0xfffffff0);  // vector count (in bytes)
-      masm.andl(rax, 0x0000000e);  // tail count (in bytes)
-      masm.testl(rcx, rcx);
-      masm.jccb(Assembler::zero, COMPARE_TAIL);
-      masm.lea(rdi, Address(rdi, rcx, Address::times_1));
-      masm.lea(rsi, Address(rsi, rcx, Address::times_1));
-      masm.negl(rcx);
-
-      masm.bind(COMPARE_WIDE_VECTORS);
-      masm.movdqu(tmp1Reg, Address(rdi, rcx, Address::times_1));
-      masm.movdqu(tmp2Reg, Address(rsi, rcx, Address::times_1));
-      masm.pxor(tmp1Reg, tmp2Reg);
-      masm.ptest(tmp1Reg, tmp1Reg);
-      masm.jccb(Assembler::notZero, RET_FALSE);
-      masm.addl(rcx, 16);
-      masm.jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
-      masm.bind(COMPARE_TAIL);
-      masm.movl(rcx, rax);
-      // Fallthru to tail compare
-    }
-
-    // Compare 4-byte vectors
-    masm.andl(rcx, 0xfffffffc);  // vector count (in bytes)
-    masm.andl(rax, 0x00000002);  // tail char (in bytes)
-    masm.testl(rcx, rcx);
-    masm.jccb(Assembler::zero, COMPARE_CHAR);
-    masm.lea(rdi, Address(rdi, rcx, Address::times_1));
-    masm.lea(rsi, Address(rsi, rcx, Address::times_1));
-    masm.negl(rcx);
-
-    masm.bind(COMPARE_VECTORS);
-    masm.movl(rbx, Address(rdi, rcx, Address::times_1));
-    masm.cmpl(rbx, Address(rsi, rcx, Address::times_1));
-    masm.jccb(Assembler::notEqual, RET_FALSE);
-    masm.addl(rcx, 4);
-    masm.jcc(Assembler::notZero, COMPARE_VECTORS);
-
-    // Compare trailing char (final 2 bytes), if any
-    masm.bind(COMPARE_CHAR);
-    masm.testl(rax, rax);
-    masm.jccb(Assembler::zero, RET_TRUE);
-    masm.load_unsigned_short(rbx, Address(rdi, 0));
-    masm.load_unsigned_short(rcx, Address(rsi, 0));
-    masm.cmpl(rbx, rcx);
-    masm.jccb(Assembler::notEqual, RET_FALSE);
-
-    masm.bind(RET_TRUE);
-    masm.movl(rax, 1);   // return true
-    masm.jmpb(DONE);
-
-    masm.bind(RET_FALSE);
-    masm.xorl(rax, rax); // return false
-
-    masm.bind(DONE);
-    %}
-
- enc_class enc_String_IndexOf(eSIRegP str1, eDIRegP str2, regXD tmp1, eAXRegI tmp2,
-                        eCXRegI tmp3, eDXRegI tmp4, eBXRegI result) %{
-    // SSE4.2 version
-    Label LOAD_SUBSTR, PREP_FOR_SCAN, SCAN_TO_SUBSTR,
-          SCAN_SUBSTR, RET_NEG_ONE, RET_NOT_FOUND, CLEANUP, DONE;
-    MacroAssembler masm(&cbuf);
-
-    XMMRegister tmp1Reg   = as_XMMRegister($tmp1$$reg);
-
-    // Get the first character position in both strings
-    //         [8] char array, [12] offset, [16] count
-    int value_offset  = java_lang_String::value_offset_in_bytes();
-    int offset_offset = java_lang_String::offset_offset_in_bytes();
-    int count_offset  = java_lang_String::count_offset_in_bytes();
-    int base_offset   = arrayOopDesc::base_offset_in_bytes(T_CHAR);
-
-    // Get counts for string and substr
-    masm.movl(rdx, Address(rsi, count_offset));
-    masm.movl(rax, Address(rdi, count_offset));
-    // Check for substr count > string count
-    masm.cmpl(rax, rdx);
-    masm.jcc(Assembler::greater, RET_NEG_ONE);
-
-    // Start the indexOf operation
-    // Get start addr of string
-    masm.movptr(rbx, Address(rsi, value_offset));
-    masm.movl(rcx, Address(rsi, offset_offset));
-    masm.lea(rsi, Address(rbx, rcx, Address::times_2, base_offset));
-    masm.push(rsi);
-
-    // Get start addr of substr
-    masm.movptr(rbx, Address(rdi, value_offset));
-    masm.movl(rcx, Address(rdi, offset_offset));
-    masm.lea(rdi, Address(rbx, rcx, Address::times_2, base_offset));
-    masm.push(rdi);
-    masm.push(rax);
-    masm.jmpb(PREP_FOR_SCAN);
-
-    // Substr count saved at sp
-    // Substr saved at sp+4
-    // String saved at sp+8
-
-    // Prep to load substr for scan
-    masm.bind(LOAD_SUBSTR);
-    masm.movptr(rdi, Address(rsp, 4));
-    masm.movl(rax, Address(rsp, 0));
-
-    // Load substr
-    masm.bind(PREP_FOR_SCAN);
-    masm.movdqu(tmp1Reg, Address(rdi, 0));
-    masm.addl(rdx, 8);        // prime the loop
-    masm.subptr(rsi, 16);
-
-    // Scan string for substr in 16-byte vectors
-    masm.bind(SCAN_TO_SUBSTR);
-    masm.subl(rdx, 8);
-    masm.addptr(rsi, 16);
-    masm.pcmpestri(tmp1Reg, Address(rsi, 0), 0x0d);
-    masm.jcc(Assembler::above, SCAN_TO_SUBSTR);     // CF == 0 && ZF == 0
-    masm.jccb(Assembler::aboveEqual, RET_NOT_FOUND); // CF == 0
-
-    // Fallthru: found a potential substr
-
-    // Make sure string is still long enough
-    masm.subl(rdx, rcx);
-    masm.cmpl(rdx, rax);
-    masm.jccb(Assembler::negative, RET_NOT_FOUND);
-    // Compute start addr of substr
-    masm.lea(rsi, Address(rsi, rcx, Address::times_2));
-    masm.movptr(rbx, rsi);
-
-    // Compare potential substr
-    masm.addl(rdx, 8);        // prime the loop
-    masm.addl(rax, 8);
-    masm.subptr(rsi, 16);
-    masm.subptr(rdi, 16);
-
-    // Scan 16-byte vectors of string and substr
-    masm.bind(SCAN_SUBSTR);
-    masm.subl(rax, 8);
-    masm.subl(rdx, 8);
-    masm.addptr(rsi, 16);
-    masm.addptr(rdi, 16);
-    masm.movdqu(tmp1Reg, Address(rdi, 0));
-    masm.pcmpestri(tmp1Reg, Address(rsi, 0), 0x0d);
-    masm.jcc(Assembler::noOverflow, LOAD_SUBSTR);   // OF == 0
-    masm.jcc(Assembler::positive, SCAN_SUBSTR);     // SF == 0
-
-    // Compute substr offset
-    masm.movptr(rsi, Address(rsp, 8));
-    masm.subptr(rbx, rsi);
-    masm.shrl(rbx, 1);
-    masm.jmpb(CLEANUP);
-
-    masm.bind(RET_NEG_ONE);
-    masm.movl(rbx, -1);
-    masm.jmpb(DONE);
-
-    masm.bind(RET_NOT_FOUND);
-    masm.movl(rbx, -1);
-
-    masm.bind(CLEANUP);
-    masm.addptr(rsp, 12);
-
-    masm.bind(DONE);
-  %}
-
-  enc_class enc_Array_Equals(eDIRegP ary1, eSIRegP ary2, regXD tmp1, regXD tmp2,
-                             eBXRegI tmp3, eDXRegI tmp4, eAXRegI result) %{
-    Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR;
-    MacroAssembler masm(&cbuf);
-
-    XMMRegister tmp1Reg   = as_XMMRegister($tmp1$$reg);
-    XMMRegister tmp2Reg   = as_XMMRegister($tmp2$$reg);
-    Register ary1Reg      = as_Register($ary1$$reg);
-    Register ary2Reg      = as_Register($ary2$$reg);
-    Register tmp3Reg      = as_Register($tmp3$$reg);
-    Register tmp4Reg      = as_Register($tmp4$$reg);
-    Register resultReg    = as_Register($result$$reg);
-
-    int length_offset  = arrayOopDesc::length_offset_in_bytes();
-    int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
-
-    // Check the input args
-    masm.cmpptr(ary1Reg, ary2Reg);
-    masm.jcc(Assembler::equal, TRUE_LABEL);
-    masm.testptr(ary1Reg, ary1Reg);
-    masm.jcc(Assembler::zero, FALSE_LABEL);
-    masm.testptr(ary2Reg, ary2Reg);
-    masm.jcc(Assembler::zero, FALSE_LABEL);
-
-    // Check the lengths
-    masm.movl(tmp4Reg, Address(ary1Reg, length_offset));
-    masm.movl(resultReg, Address(ary2Reg, length_offset));
-    masm.cmpl(tmp4Reg, resultReg);
-    masm.jcc(Assembler::notEqual, FALSE_LABEL);
-    masm.testl(resultReg, resultReg);
-    masm.jcc(Assembler::zero, TRUE_LABEL);
-
-    // Load array addrs
-    masm.lea(ary1Reg, Address(ary1Reg, base_offset));
-    masm.lea(ary2Reg, Address(ary2Reg, base_offset));
-
-    // Set byte count
-    masm.shll(tmp4Reg, 1);
-    masm.movl(resultReg, tmp4Reg);
-
-    if (UseSSE42Intrinsics) {
-      // With SSE4.2, use double quad vector compare
-      Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
-      // Compare 16-byte vectors
-      masm.andl(tmp4Reg, 0xfffffff0);    // vector count (in bytes)
-      masm.andl(resultReg, 0x0000000e);  // tail count (in bytes)
-      masm.testl(tmp4Reg, tmp4Reg);
-      masm.jccb(Assembler::zero, COMPARE_TAIL);
-      masm.lea(ary1Reg, Address(ary1Reg, tmp4Reg, Address::times_1));
-      masm.lea(ary2Reg, Address(ary2Reg, tmp4Reg, Address::times_1));
-      masm.negl(tmp4Reg);
-
-      masm.bind(COMPARE_WIDE_VECTORS);
-      masm.movdqu(tmp1Reg, Address(ary1Reg, tmp4Reg, Address::times_1));
-      masm.movdqu(tmp2Reg, Address(ary2Reg, tmp4Reg, Address::times_1));
-      masm.pxor(tmp1Reg, tmp2Reg);
-      masm.ptest(tmp1Reg, tmp1Reg);
-
-      masm.jccb(Assembler::notZero, FALSE_LABEL);
-      masm.addl(tmp4Reg, 16);
-      masm.jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
-      masm.bind(COMPARE_TAIL);
-      masm.movl(tmp4Reg, resultReg);
-      // Fallthru to tail compare
-    }
-
-    // Compare 4-byte vectors
-    masm.andl(tmp4Reg, 0xfffffffc);    // vector count (in bytes)
-    masm.andl(resultReg, 0x00000002);  // tail char (in bytes)
-    masm.testl(tmp4Reg, tmp4Reg);
-    masm.jccb(Assembler::zero, COMPARE_CHAR);
-    masm.lea(ary1Reg, Address(ary1Reg, tmp4Reg, Address::times_1));
-    masm.lea(ary2Reg, Address(ary2Reg, tmp4Reg, Address::times_1));
-    masm.negl(tmp4Reg);
-
-    masm.bind(COMPARE_VECTORS);
-    masm.movl(tmp3Reg, Address(ary1Reg, tmp4Reg, Address::times_1));
-    masm.cmpl(tmp3Reg, Address(ary2Reg, tmp4Reg, Address::times_1));
-    masm.jccb(Assembler::notEqual, FALSE_LABEL);
-    masm.addl(tmp4Reg, 4);
-    masm.jcc(Assembler::notZero, COMPARE_VECTORS);
-
-    // Compare trailing char (final 2 bytes), if any
-    masm.bind(COMPARE_CHAR);
-    masm.testl(resultReg, resultReg);
-    masm.jccb(Assembler::zero, TRUE_LABEL);
-    masm.load_unsigned_short(tmp3Reg, Address(ary1Reg, 0));
-    masm.load_unsigned_short(tmp4Reg, Address(ary2Reg, 0));
-    masm.cmpl(tmp3Reg, tmp4Reg);
-    masm.jccb(Assembler::notEqual, FALSE_LABEL);
-
-    masm.bind(TRUE_LABEL);
-    masm.movl(resultReg, 1);   // return true
-    masm.jmpb(DONE);
-
-    masm.bind(FALSE_LABEL);
-    masm.xorl(resultReg, resultReg); // return false
-
-    // That's it
-    masm.bind(DONE);
-  %}
 
   enc_class enc_pop_rdx() %{
     emit_opcode(cbuf,0x5A);
@@ -12718,48 +12266,64 @@
   ins_pipe( pipe_slow );
 %}
 
-instruct string_compare(eDIRegP str1, eSIRegP str2, regXD tmp1, regXD tmp2,
-                        eAXRegI tmp3, eBXRegI tmp4, eCXRegI result, eFlagsReg cr) %{
-  match(Set result (StrComp str1 str2));
-  effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, KILL tmp3, KILL tmp4, KILL cr);
-  //ins_cost(300);
-
-  format %{ "String Compare $str1,$str2 -> $result    // KILL EAX, EBX" %}
-  ins_encode( enc_String_Compare(str1, str2, tmp1, tmp2, tmp3, tmp4, result) );
+instruct string_compare(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eBXRegI cnt2,
+                        eAXRegI result, regXD tmp1, regXD tmp2, eFlagsReg cr) %{
+  match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
+  effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
+
+  format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1, $tmp2" %}
+  ins_encode %{
+    __ string_compare($str1$$Register, $str2$$Register,
+                      $cnt1$$Register, $cnt2$$Register, $result$$Register,
+                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
+  %}
   ins_pipe( pipe_slow );
 %}
 
 // fast string equals
-instruct string_equals(eDIRegP str1, eSIRegP str2, regXD tmp1, regXD tmp2,
-                       eBXRegI tmp3, eCXRegI tmp4, eAXRegI result, eFlagsReg cr) %{
-  match(Set result (StrEquals str1 str2));
-  effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, KILL tmp3, KILL tmp4, KILL cr);
-
-  format %{ "String Equals $str1,$str2 -> $result    // KILL EBX, ECX" %}
-  ins_encode( enc_String_Equals(tmp1, tmp2, str1, str2, tmp3, tmp4, result) );
-  ins_pipe( pipe_slow );
-%}
-
-instruct string_indexof(eSIRegP str1, eDIRegP str2, regXD tmp1, eAXRegI tmp2,
-                        eCXRegI tmp3, eDXRegI tmp4, eBXRegI result, eFlagsReg cr) %{
+instruct string_equals(eDIRegP str1, eSIRegP str2, eCXRegI cnt, eAXRegI result,
+                       regXD tmp1, regXD tmp2, eBXRegI tmp3, eFlagsReg cr) %{
+  match(Set result (StrEquals (Binary str1 str2) cnt));
+  effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
+
+  format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
+  ins_encode %{
+    __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
+                          $cnt$$Register, $result$$Register, $tmp3$$Register,
+                          $tmp1$$XMMRegister, $tmp2$$XMMRegister);
+  %}
+  ins_pipe( pipe_slow );
+%}
+
+instruct string_indexof(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, eAXRegI cnt2,
+                        eBXRegI result, regXD tmp1, eCXRegI tmp2, eFlagsReg cr) %{
   predicate(UseSSE42Intrinsics);
-  match(Set result (StrIndexOf str1 str2));
-  effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, KILL tmp2, KILL tmp3, KILL tmp4, KILL cr);
-
-  format %{ "String IndexOf $str1,$str2 -> $result    // KILL EAX, ECX, EDX" %}
-  ins_encode( enc_String_IndexOf(str1, str2, tmp1, tmp2, tmp3, tmp4, result) );
+  match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
+  effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp2, KILL cr);
+
+  format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp2, $tmp1" %}
+  ins_encode %{
+    __ string_indexof($str1$$Register, $str2$$Register,
+                      $cnt1$$Register, $cnt2$$Register, $result$$Register,
+                      $tmp1$$XMMRegister, $tmp2$$Register);
+  %}
   ins_pipe( pipe_slow );
 %}
 
 // fast array equals
-instruct array_equals(eDIRegP ary1, eSIRegP ary2, regXD tmp1, regXD tmp2, eBXRegI tmp3,
-                      eDXRegI tmp4, eAXRegI result, eFlagsReg cr) %{
+instruct array_equals(eDIRegP ary1, eSIRegP ary2, eAXRegI result,
+                      regXD tmp1, regXD tmp2, eCXRegI tmp3, eBXRegI tmp4, eFlagsReg cr)
+%{
   match(Set result (AryEq ary1 ary2));
   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
   //ins_cost(300);
 
-  format %{ "Array Equals $ary1,$ary2 -> $result    // KILL EBX, EDX" %}
-  ins_encode( enc_Array_Equals(ary1, ary2, tmp1, tmp2, tmp3, tmp4, result) );
+  format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
+  ins_encode %{
+    __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
+                          $tmp3$$Register, $result$$Register, $tmp4$$Register,
+                          $tmp1$$XMMRegister, $tmp2$$XMMRegister);
+  %}
   ins_pipe( pipe_slow );
 %}
 

--- a/src/cpu/x86/vm/x86_64.ad	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/cpu/x86/vm/x86_64.ad	Wed Sep 30 13:25:39 2009 -0700
@@ -683,7 +683,7 @@
 #ifdef ASSERT
   if (rspec.reloc()->type() == relocInfo::oop_type &&
       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
-    assert(oop((intptr_t)d32)->is_oop() && oop((intptr_t)d32)->is_perm(), "cannot embed non-perm oops in code");
+    assert(oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
   }
 #endif
   cbuf.relocate(cbuf.inst_mark(), rspec, format);
@@ -721,8 +721,8 @@
 #ifdef ASSERT
   if (rspec.reloc()->type() == relocInfo::oop_type &&
       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
-    assert(oop(d64)->is_oop() && oop(d64)->is_perm(),
-           "cannot embed non-perm oops in code");
+    assert(oop(d64)->is_oop() && (ScavengeRootsInCode || !oop(d64)->is_scavengable()),
+           "cannot embed scavengable oops in code");
   }
 #endif
   cbuf.relocate(cbuf.inst_mark(), rspec, format);
@@ -3701,448 +3701,6 @@
     }
   %}
 
-  enc_class enc_String_Compare(rdi_RegP str1, rsi_RegP str2, regD tmp1, regD tmp2,
-                        rax_RegI tmp3, rbx_RegI tmp4, rcx_RegI result) %{
-    Label RCX_GOOD_LABEL, LENGTH_DIFF_LABEL,
-          POP_LABEL, DONE_LABEL, CONT_LABEL,
-          WHILE_HEAD_LABEL;
-    MacroAssembler masm(&cbuf);
-
-    XMMRegister tmp1Reg   = as_XMMRegister($tmp1$$reg);
-    XMMRegister tmp2Reg   = as_XMMRegister($tmp2$$reg);
-
-    // Get the first character position in both strings
-    //         [8] char array, [12] offset, [16] count
-    int value_offset  = java_lang_String::value_offset_in_bytes();
-    int offset_offset = java_lang_String::offset_offset_in_bytes();
-    int count_offset  = java_lang_String::count_offset_in_bytes();
-    int base_offset   = arrayOopDesc::base_offset_in_bytes(T_CHAR);
-
-    masm.load_heap_oop(rax, Address(rsi, value_offset));
-    masm.movl(rcx, Address(rsi, offset_offset));
-    masm.lea(rax, Address(rax, rcx, Address::times_2, base_offset));
-    masm.load_heap_oop(rbx, Address(rdi, value_offset));
-    masm.movl(rcx, Address(rdi, offset_offset));
-    masm.lea(rbx, Address(rbx, rcx, Address::times_2, base_offset));
-
-    // Compute the minimum of the string lengths(rsi) and the
-    // difference of the string lengths (stack)
-
-    // do the conditional move stuff
-    masm.movl(rdi, Address(rdi, count_offset));
-    masm.movl(rsi, Address(rsi, count_offset));
-    masm.movl(rcx, rdi);
-    masm.subl(rdi, rsi);
-    masm.push(rdi);
-    masm.cmov(Assembler::lessEqual, rsi, rcx);
-
-    // Is the minimum length zero?
-    masm.bind(RCX_GOOD_LABEL);
-    masm.testl(rsi, rsi);
-    masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL);
-
-    // Load first characters
-    masm.load_unsigned_short(rcx, Address(rbx, 0));
-    masm.load_unsigned_short(rdi, Address(rax, 0));
-
-    // Compare first characters
-    masm.subl(rcx, rdi);
-    masm.jcc(Assembler::notZero,  POP_LABEL);
-    masm.decrementl(rsi);
-    masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL);
-
-    {
-      // Check after comparing first character to see if strings are equivalent
-      Label LSkip2;
-      // Check if the strings start at same location
-      masm.cmpptr(rbx, rax);
-      masm.jccb(Assembler::notEqual, LSkip2);
-
-      // Check if the length difference is zero (from stack)
-      masm.cmpl(Address(rsp, 0), 0x0);
-      masm.jcc(Assembler::equal,  LENGTH_DIFF_LABEL);
-
-      // Strings might not be equivalent
-      masm.bind(LSkip2);
-    }
-
-    // Advance to next character
-    masm.addptr(rax, 2);
-    masm.addptr(rbx, 2);
-
-    if (UseSSE42Intrinsics) {
-      // With SSE4.2, use double quad vector compare
-      Label COMPARE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
-      // Setup to compare 16-byte vectors
-      masm.movl(rdi, rsi);
-      masm.andl(rsi, 0xfffffff8); // rsi holds the vector count
-      masm.andl(rdi, 0x00000007); // rdi holds the tail count
-      masm.testl(rsi, rsi);
-      masm.jccb(Assembler::zero, COMPARE_TAIL);
-
-      masm.lea(rax, Address(rax, rsi, Address::times_2));
-      masm.lea(rbx, Address(rbx, rsi, Address::times_2));
-      masm.negptr(rsi);
-
-      masm.bind(COMPARE_VECTORS);
-      masm.movdqu(tmp1Reg, Address(rax, rsi, Address::times_2));
-      masm.movdqu(tmp2Reg, Address(rbx, rsi, Address::times_2));
-      masm.pxor(tmp1Reg, tmp2Reg);
-      masm.ptest(tmp1Reg, tmp1Reg);
-      masm.jccb(Assembler::notZero, VECTOR_NOT_EQUAL);
-      masm.addptr(rsi, 8);
-      masm.jcc(Assembler::notZero, COMPARE_VECTORS);
-      masm.jmpb(COMPARE_TAIL);
-
-      // Mismatched characters in the vectors
-      masm.bind(VECTOR_NOT_EQUAL);
-      masm.lea(rax, Address(rax, rsi, Address::times_2));
-      masm.lea(rbx, Address(rbx, rsi, Address::times_2));
-      masm.movl(rdi, 8);
-
-      // Compare tail (< 8 chars), or rescan last vectors to
-      // find 1st mismatched characters
-      masm.bind(COMPARE_TAIL);
-      masm.testl(rdi, rdi);
-      masm.jccb(Assembler::zero, LENGTH_DIFF_LABEL);
-      masm.movl(rsi, rdi);
-      // Fallthru to tail compare
-    }
-
-    // Shift RAX and RBX to the end of the arrays, negate min
-    masm.lea(rax, Address(rax, rsi, Address::times_2, 0));
-    masm.lea(rbx, Address(rbx, rsi, Address::times_2, 0));
-    masm.negptr(rsi);
-
-    // Compare the rest of the characters
-    masm.bind(WHILE_HEAD_LABEL);
-    masm.load_unsigned_short(rcx, Address(rbx, rsi, Address::times_2, 0));
-    masm.load_unsigned_short(rdi, Address(rax, rsi, Address::times_2, 0));
-    masm.subl(rcx, rdi);
-    masm.jccb(Assembler::notZero, POP_LABEL);
-    masm.increment(rsi);
-    masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL);
-
-    // Strings are equal up to min length.  Return the length difference.
-    masm.bind(LENGTH_DIFF_LABEL);
-    masm.pop(rcx);
-    masm.jmpb(DONE_LABEL);
-
-    // Discard the stored length difference
-    masm.bind(POP_LABEL);
-    masm.addptr(rsp, 8);
-
-    // That's it
-    masm.bind(DONE_LABEL);
-  %}
-
- enc_class enc_String_IndexOf(rsi_RegP str1, rdi_RegP str2, regD tmp1, rax_RegI tmp2,
-                        rcx_RegI tmp3, rdx_RegI tmp4, rbx_RegI result) %{
-    // SSE4.2 version
-    Label LOAD_SUBSTR, PREP_FOR_SCAN, SCAN_TO_SUBSTR,
-          SCAN_SUBSTR, RET_NEG_ONE, RET_NOT_FOUND, CLEANUP, DONE;
-    MacroAssembler masm(&cbuf);
-
-    XMMRegister tmp1Reg   = as_XMMRegister($tmp1$$reg);
-
-    // Get the first character position in both strings
-    //         [8] char array, [12] offset, [16] count
-    int value_offset  = java_lang_String::value_offset_in_bytes();
-    int offset_offset = java_lang_String::offset_offset_in_bytes();
-    int count_offset  = java_lang_String::count_offset_in_bytes();
-    int base_offset   = arrayOopDesc::base_offset_in_bytes(T_CHAR);
-
-    // Get counts for string and substr
-    masm.movl(rdx, Address(rsi, count_offset));
-    masm.movl(rax, Address(rdi, count_offset));
-    // Check for substr count > string count
-    masm.cmpl(rax, rdx);
-    masm.jcc(Assembler::greater, RET_NEG_ONE);
-
-    // Start the indexOf operation
-    // Get start addr of string
-    masm.load_heap_oop(rbx, Address(rsi, value_offset));
-    masm.movl(rcx, Address(rsi, offset_offset));
-    masm.lea(rsi, Address(rbx, rcx, Address::times_2, base_offset));
-    masm.push(rsi);
-
-    // Get start addr of substr
-    masm.load_heap_oop(rbx, Address(rdi, value_offset));
-    masm.movl(rcx, Address(rdi, offset_offset));
-    masm.lea(rdi, Address(rbx, rcx, Address::times_2, base_offset));
-    masm.push(rdi);
-    masm.push(rax);
-    masm.jmpb(PREP_FOR_SCAN);
-
-    // Substr count saved at sp
-    // Substr saved at sp+8
-    // String saved at sp+16
-
-    // Prep to load substr for scan
-    masm.bind(LOAD_SUBSTR);
-    masm.movptr(rdi, Address(rsp, 8));
-    masm.movl(rax, Address(rsp, 0));
-
-    // Load substr
-    masm.bind(PREP_FOR_SCAN);
-    masm.movdqu(tmp1Reg, Address(rdi, 0));
-    masm.addq(rdx, 8);    // prime the loop
-    masm.subptr(rsi, 16);
-
-    // Scan string for substr in 16-byte vectors
-    masm.bind(SCAN_TO_SUBSTR);
-    masm.subq(rdx, 8);
-    masm.addptr(rsi, 16);
-    masm.pcmpestri(tmp1Reg, Address(rsi, 0), 0x0d);
-    masm.jcc(Assembler::above, SCAN_TO_SUBSTR);
-    masm.jccb(Assembler::aboveEqual, RET_NOT_FOUND);
-
-    // Fallthru: found a potential substr
-
-    //Make sure string is still long enough
-    masm.subl(rdx, rcx);
-    masm.cmpl(rdx, rax);
-    masm.jccb(Assembler::negative, RET_NOT_FOUND);
-    // Compute start addr of substr
-    masm.lea(rsi, Address(rsi, rcx, Address::times_2));
-    masm.movptr(rbx, rsi);
-
-    // Compare potential substr
-    masm.addq(rdx, 8);        // prime the loop
-    masm.addq(rax, 8);
-    masm.subptr(rsi, 16);
-    masm.subptr(rdi, 16);
-
-    // Scan 16-byte vectors of string and substr
-    masm.bind(SCAN_SUBSTR);
-    masm.subq(rax, 8);
-    masm.subq(rdx, 8);
-    masm.addptr(rsi, 16);
-    masm.addptr(rdi, 16);
-    masm.movdqu(tmp1Reg, Address(rdi, 0));
-    masm.pcmpestri(tmp1Reg, Address(rsi, 0), 0x0d);
-    masm.jcc(Assembler::noOverflow, LOAD_SUBSTR);   // OF == 0
-    masm.jcc(Assembler::positive, SCAN_SUBSTR);     // SF == 0
-
-    // Compute substr offset
-    masm.movptr(rsi, Address(rsp, 16));
-    masm.subptr(rbx, rsi);
-    masm.shrl(rbx, 1);
-    masm.jmpb(CLEANUP);
-
-    masm.bind(RET_NEG_ONE);
-    masm.movl(rbx, -1);
-    masm.jmpb(DONE);
-
-    masm.bind(RET_NOT_FOUND);
-    masm.movl(rbx, -1);
-
-    masm.bind(CLEANUP);
-    masm.addptr(rsp, 24);
-
-    masm.bind(DONE);
-  %}
-
-  enc_class enc_String_Equals(rdi_RegP str1, rsi_RegP str2, regD tmp1, regD tmp2,
-                              rbx_RegI tmp3, rcx_RegI tmp2, rax_RegI result) %{
-    Label RET_TRUE, RET_FALSE, DONE, COMPARE_VECTORS, COMPARE_CHAR;
-    MacroAssembler masm(&cbuf);
-
-    XMMRegister tmp1Reg   = as_XMMRegister($tmp1$$reg);
-    XMMRegister tmp2Reg   = as_XMMRegister($tmp2$$reg);
-
-    int value_offset  = java_lang_String::value_offset_in_bytes();
-    int offset_offset = java_lang_String::offset_offset_in_bytes();
-    int count_offset  = java_lang_String::count_offset_in_bytes();
-    int base_offset   = arrayOopDesc::base_offset_in_bytes(T_CHAR);
-
-    // does source == target string?
-    masm.cmpptr(rdi, rsi);
-    masm.jcc(Assembler::equal, RET_TRUE);
-
-    // get and compare counts
-    masm.movl(rcx, Address(rdi, count_offset));
-    masm.movl(rax, Address(rsi, count_offset));
-    masm.cmpl(rcx, rax);
-    masm.jcc(Assembler::notEqual, RET_FALSE);
-    masm.testl(rax, rax);
-    masm.jcc(Assembler::zero, RET_TRUE);
-
-    // get source string offset and value
-    masm.load_heap_oop(rbx, Address(rsi, value_offset));
-    masm.movl(rax, Address(rsi, offset_offset));
-    masm.lea(rsi, Address(rbx, rax, Address::times_2, base_offset));
-
-    // get compare string offset and value
-    masm.load_heap_oop(rbx, Address(rdi, value_offset));
-    masm.movl(rax, Address(rdi, offset_offset));
-    masm.lea(rdi, Address(rbx, rax, Address::times_2, base_offset));
-
-    // Set byte count
-    masm.shll(rcx, 1);
-    masm.movl(rax, rcx);
-
-    if (UseSSE42Intrinsics) {
-      // With SSE4.2, use double quad vector compare
-      Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
-      // Compare 16-byte vectors
-      masm.andl(rcx, 0xfffffff0);  // vector count (in bytes)
-      masm.andl(rax, 0x0000000e);  // tail count (in bytes)
-      masm.testl(rcx, rcx);
-      masm.jccb(Assembler::zero, COMPARE_TAIL);
-      masm.lea(rdi, Address(rdi, rcx, Address::times_1));
-      masm.lea(rsi, Address(rsi, rcx, Address::times_1));
-      masm.negptr(rcx);
-
-      masm.bind(COMPARE_WIDE_VECTORS);
-      masm.movdqu(tmp1Reg, Address(rdi, rcx, Address::times_1));
-      masm.movdqu(tmp2Reg, Address(rsi, rcx, Address::times_1));
-      masm.pxor(tmp1Reg, tmp2Reg);
-      masm.ptest(tmp1Reg, tmp1Reg);
-      masm.jccb(Assembler::notZero, RET_FALSE);
-      masm.addptr(rcx, 16);
-      masm.jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
-      masm.bind(COMPARE_TAIL);
-      masm.movl(rcx, rax);
-      // Fallthru to tail compare
-    }
-
-    // Compare 4-byte vectors
-    masm.andl(rcx, 0xfffffffc);  // vector count (in bytes)
-    masm.andl(rax, 0x00000002);  // tail char (in bytes)
-    masm.testl(rcx, rcx);
-    masm.jccb(Assembler::zero, COMPARE_CHAR);
-    masm.lea(rdi, Address(rdi, rcx, Address::times_1));
-    masm.lea(rsi, Address(rsi, rcx, Address::times_1));
-    masm.negptr(rcx);
-
-    masm.bind(COMPARE_VECTORS);
-    masm.movl(rbx, Address(rdi, rcx, Address::times_1));
-    masm.cmpl(rbx, Address(rsi, rcx, Address::times_1));
-    masm.jccb(Assembler::notEqual, RET_FALSE);
-    masm.addptr(rcx, 4);
-    masm.jcc(Assembler::notZero, COMPARE_VECTORS);
-
-    // Compare trailing char (final 2 bytes), if any
-    masm.bind(COMPARE_CHAR);
-    masm.testl(rax, rax);
-    masm.jccb(Assembler::zero, RET_TRUE);
-    masm.load_unsigned_short(rbx, Address(rdi, 0));
-    masm.load_unsigned_short(rcx, Address(rsi, 0));
-    masm.cmpl(rbx, rcx);
-    masm.jccb(Assembler::notEqual, RET_FALSE);
-
-    masm.bind(RET_TRUE);
-    masm.movl(rax, 1);   // return true
-    masm.jmpb(DONE);
-
-    masm.bind(RET_FALSE);
-    masm.xorl(rax, rax); // return false
-
-    masm.bind(DONE);
-  %}
-
-  enc_class enc_Array_Equals(rdi_RegP ary1, rsi_RegP ary2, regD tmp1, regD tmp2,
-                             rax_RegI tmp3, rbx_RegI tmp4, rcx_RegI result) %{
-    Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR;
-    MacroAssembler masm(&cbuf);
-
-    XMMRegister tmp1Reg   = as_XMMRegister($tmp1$$reg);
-    XMMRegister tmp2Reg   = as_XMMRegister($tmp2$$reg);
-    Register ary1Reg      = as_Register($ary1$$reg);
-    Register ary2Reg      = as_Register($ary2$$reg);
-    Register tmp3Reg      = as_Register($tmp3$$reg);
-    Register tmp4Reg      = as_Register($tmp4$$reg);
-    Register resultReg    = as_Register($result$$reg);
-
-    int length_offset  = arrayOopDesc::length_offset_in_bytes();
-    int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
-
-    // Check the input args
-    masm.cmpq(ary1Reg, ary2Reg);
-    masm.jcc(Assembler::equal, TRUE_LABEL);
-    masm.testq(ary1Reg, ary1Reg);
-    masm.jcc(Assembler::zero, FALSE_LABEL);
-    masm.testq(ary2Reg, ary2Reg);
-    masm.jcc(Assembler::zero, FALSE_LABEL);
-
-    // Check the lengths
-    masm.movl(tmp4Reg, Address(ary1Reg, length_offset));
-    masm.movl(resultReg, Address(ary2Reg, length_offset));
-    masm.cmpl(tmp4Reg, resultReg);
-    masm.jcc(Assembler::notEqual, FALSE_LABEL);
-    masm.testl(resultReg, resultReg);
-    masm.jcc(Assembler::zero, TRUE_LABEL);
-
-    //load array address
-    masm.lea(ary1Reg, Address(ary1Reg, base_offset));
-    masm.lea(ary2Reg, Address(ary2Reg, base_offset));
-
-    //set byte count
-    masm.shll(tmp4Reg, 1);
-    masm.movl(resultReg,tmp4Reg);
-
-    if (UseSSE42Intrinsics){
-      // With SSE4.2, use double quad vector compare
-      Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
-      // Compare 16-byte vectors
-      masm.andl(tmp4Reg, 0xfffffff0);    // vector count (in bytes)
-      masm.andl(resultReg, 0x0000000e);  // tail count (in bytes)
-      masm.testl(tmp4Reg, tmp4Reg);
-      masm.jccb(Assembler::zero, COMPARE_TAIL);
-      masm.lea(ary1Reg, Address(ary1Reg, tmp4Reg, Address::times_1));
-      masm.lea(ary2Reg, Address(ary2Reg, tmp4Reg, Address::times_1));
-      masm.negptr(tmp4Reg);
-
-      masm.bind(COMPARE_WIDE_VECTORS);
-      masm.movdqu(tmp1Reg, Address(ary1Reg, tmp4Reg, Address::times_1));
-      masm.movdqu(tmp2Reg, Address(ary2Reg, tmp4Reg, Address::times_1));
-      masm.pxor(tmp1Reg, tmp2Reg);
-      masm.ptest(tmp1Reg, tmp1Reg);
-
-      masm.jccb(Assembler::notZero, FALSE_LABEL);
-      masm.addptr(tmp4Reg, 16);
-      masm.jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
-      masm.bind(COMPARE_TAIL);
-      masm.movl(tmp4Reg, resultReg);
-      // Fallthru to tail compare
-    }
-
-   // Compare 4-byte vectors
-    masm.andl(tmp4Reg, 0xfffffffc);    // vector count (in bytes)
-    masm.andl(resultReg, 0x00000002);  // tail char (in bytes)
-    masm.testl(tmp4Reg, tmp4Reg); //if tmp2 == 0, only compare char
-    masm.jccb(Assembler::zero, COMPARE_CHAR);
-    masm.lea(ary1Reg, Address(ary1Reg, tmp4Reg, Address::times_1));
-    masm.lea(ary2Reg, Address(ary2Reg, tmp4Reg, Address::times_1));
-    masm.negptr(tmp4Reg);
-
-    masm.bind(COMPARE_VECTORS);
-    masm.movl(tmp3Reg, Address(ary1Reg, tmp4Reg, Address::times_1));
-    masm.cmpl(tmp3Reg, Address(ary2Reg, tmp4Reg, Address::times_1));
-    masm.jccb(Assembler::notEqual, FALSE_LABEL);
-    masm.addptr(tmp4Reg, 4);
-    masm.jcc(Assembler::notZero, COMPARE_VECTORS);
-
-    // Compare trailing char (final 2 bytes), if any
-    masm.bind(COMPARE_CHAR);
-    masm.testl(resultReg, resultReg);
-    masm.jccb(Assembler::zero, TRUE_LABEL);
-    masm.load_unsigned_short(tmp3Reg, Address(ary1Reg, 0));
-    masm.load_unsigned_short(tmp4Reg, Address(ary2Reg, 0));
-    masm.cmpl(tmp3Reg, tmp4Reg);
-    masm.jccb(Assembler::notEqual, FALSE_LABEL);
-
-    masm.bind(TRUE_LABEL);
-    masm.movl(resultReg, 1);   // return true
-    masm.jmpb(DONE);
-
-    masm.bind(FALSE_LABEL);
-    masm.xorl(resultReg, resultReg); // return false
-
-    // That's it
-    masm.bind(DONE);
-  %}
 
   enc_class enc_rethrow()
   %{
@@ -12096,52 +11654,67 @@
   ins_pipe(pipe_slow);
 %}
 
-instruct string_compare(rdi_RegP str1, rsi_RegP str2, regD tmp1, regD tmp2,
-                        rax_RegI tmp3, rbx_RegI tmp4, rcx_RegI result, rFlagsReg cr)
-%{
-  match(Set result (StrComp str1 str2));
-  effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, KILL tmp3, KILL tmp4, KILL cr);
-  //ins_cost(300);
-
-  format %{ "String Compare $str1, $str2 -> $result    // XXX KILL RAX, RBX" %}
-  ins_encode( enc_String_Compare(str1, str2, tmp1, tmp2, tmp3, tmp4, result) );
+instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rbx_RegI cnt2,
+                        rax_RegI result, regD tmp1, regD tmp2, rFlagsReg cr)
+%{
+  match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
+  effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
+
+  format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1, $tmp2" %}
+  ins_encode %{
+    __ string_compare($str1$$Register, $str2$$Register,
+                      $cnt1$$Register, $cnt2$$Register, $result$$Register,
+                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
+  %}
   ins_pipe( pipe_slow );
 %}
 
-instruct string_indexof(rsi_RegP str1, rdi_RegP str2, regD tmp1, rax_RegI tmp2,
-                        rcx_RegI tmp3, rdx_RegI tmp4, rbx_RegI result, rFlagsReg cr)
+instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
+                        rbx_RegI result, regD tmp1, rcx_RegI tmp2, rFlagsReg cr)
 %{
   predicate(UseSSE42Intrinsics);
-  match(Set result (StrIndexOf str1 str2));
-  effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, KILL tmp2, KILL tmp3, KILL tmp4, KILL cr);
-
-  format %{ "String IndexOf $str1,$str2 -> $result   // KILL RAX, RCX, RDX" %}
-  ins_encode( enc_String_IndexOf(str1, str2, tmp1, tmp2, tmp3, tmp4, result) );
+  match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
+  effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp2, KILL cr);
+
+  format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1, $tmp2" %}
+  ins_encode %{
+    __ string_indexof($str1$$Register, $str2$$Register,
+                      $cnt1$$Register, $cnt2$$Register, $result$$Register,
+                      $tmp1$$XMMRegister, $tmp2$$Register);
+  %}
   ins_pipe( pipe_slow );
 %}
 
 // fast string equals
-instruct string_equals(rdi_RegP str1, rsi_RegP str2, regD tmp1, regD tmp2, rbx_RegI tmp3,
-                       rcx_RegI tmp4, rax_RegI result, rFlagsReg cr)
-%{
-  match(Set result (StrEquals str1 str2));
-  effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, KILL tmp3, KILL tmp4, KILL cr);
-
-  format %{ "String Equals $str1,$str2 -> $result    // KILL RBX, RCX" %}
-  ins_encode( enc_String_Equals(str1, str2, tmp1, tmp2, tmp3, tmp4, result) );
+instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
+                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
+%{
+  match(Set result (StrEquals (Binary str1 str2) cnt));
+  effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
+
+  format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
+  ins_encode %{
+    __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
+                          $cnt$$Register, $result$$Register, $tmp3$$Register,
+                          $tmp1$$XMMRegister, $tmp2$$XMMRegister);
+  %}
   ins_pipe( pipe_slow );
 %}
 
 // fast array equals
-instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, regD tmp1, regD tmp2, rax_RegI tmp3,
-                      rbx_RegI tmp4, rcx_RegI result, rFlagsReg cr)
+instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
+                      regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
 %{
   match(Set result (AryEq ary1 ary2));
   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
   //ins_cost(300);
 
-  format %{ "Array Equals $ary1,$ary2 -> $result   // KILL RAX, RBX" %}
-  ins_encode( enc_Array_Equals(ary1, ary2, tmp1, tmp2, tmp3, tmp4, result) );
+  format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
+  ins_encode %{
+    __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
+                          $tmp3$$Register, $result$$Register, $tmp4$$Register,
+                          $tmp1$$XMMRegister, $tmp2$$XMMRegister);
+  %}
   ins_pipe( pipe_slow );
 %}
 

--- a/src/share/vm/adlc/formssel.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/adlc/formssel.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -828,11 +828,13 @@
     return AdlcVMDeps::Parms;   // Skip the machine-state edges
 
   if( _matrule->_rChild &&
-      ( strcmp(_matrule->_rChild->_opType,"StrComp"   )==0 ||
+      ( strcmp(_matrule->_rChild->_opType,"AryEq"     )==0 ||
+        strcmp(_matrule->_rChild->_opType,"StrComp"   )==0 ||
         strcmp(_matrule->_rChild->_opType,"StrEquals" )==0 ||
         strcmp(_matrule->_rChild->_opType,"StrIndexOf")==0 )) {
-        // String.(compareTo/equals/indexOf) take 1 control and 4 memory edges.
-    return 5;
+        // String.(compareTo/equals/indexOf) and Arrays.equals
+        // take 1 control and 1 memory edges.
+    return 2;
   }
 
   // Check for handling of 'Memory' input/edge in the ideal world.

--- a/src/share/vm/c1/c1_GraphBuilder.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/c1/c1_GraphBuilder.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -1442,7 +1442,7 @@
         switch (field_type) {
         case T_ARRAY:
         case T_OBJECT:
-          if (field_val.as_object()->has_encoding()) {
+          if (field_val.as_object()->should_be_constant()) {
             constant =  new Constant(as_ValueType(field_val));
           }
           break;

--- a/src/share/vm/c1/c1_InstructionPrinter.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/c1/c1_InstructionPrinter.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -133,12 +133,12 @@
       ciMethod* m = (ciMethod*)value;
       output()->print("<method %s.%s>", m->holder()->name()->as_utf8(), m->name()->as_utf8());
     } else {
-      output()->print("<object 0x%x>", value->encoding());
+      output()->print("<object 0x%x>", value->constant_encoding());
     }
   } else if (type->as_InstanceConstant() != NULL) {
-    output()->print("<instance 0x%x>", type->as_InstanceConstant()->value()->encoding());
+    output()->print("<instance 0x%x>", type->as_InstanceConstant()->value()->constant_encoding());
   } else if (type->as_ArrayConstant() != NULL) {
-    output()->print("<array 0x%x>", type->as_ArrayConstant()->value()->encoding());
+    output()->print("<array 0x%x>", type->as_ArrayConstant()->value()->constant_encoding());
   } else if (type->as_ClassConstant() != NULL) {
     ciInstanceKlass* klass = type->as_ClassConstant()->value();
     if (!klass->is_loaded()) {

--- a/src/share/vm/c1/c1_LIRGenerator.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/c1/c1_LIRGenerator.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -440,7 +440,7 @@
     __ oop2reg_patch(NULL, r, info);
   } else {
     // no patching needed
-    __ oop2reg(obj->encoding(), r);
+    __ oop2reg(obj->constant_encoding(), r);
   }
 }
 
@@ -831,7 +831,7 @@
     int taken_count_offset     = md->byte_offset_of_slot(data, BranchData::taken_offset());
     int not_taken_count_offset = md->byte_offset_of_slot(data, BranchData::not_taken_offset());
     LIR_Opr md_reg = new_register(T_OBJECT);
-    __ move(LIR_OprFact::oopConst(md->encoding()), md_reg);
+    __ move(LIR_OprFact::oopConst(md->constant_encoding()), md_reg);
     LIR_Opr data_offset_reg = new_register(T_INT);
     __ cmove(lir_cond(cond),
              LIR_OprFact::intConst(taken_count_offset),
@@ -1071,7 +1071,7 @@
     LIR_OprList* args = new LIR_OprList();
     args->append(getThreadPointer());
     LIR_Opr meth = new_register(T_OBJECT);
-    __ oop2reg(method()->encoding(), meth);
+    __ oop2reg(method()->constant_encoding(), meth);
     args->append(meth);
     call_runtime(&signature, args, CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), voidType, NULL);
   }
@@ -1784,7 +1784,7 @@
     LIR_OprList* args = new LIR_OprList();
     args->append(getThreadPointer());
     LIR_Opr meth = new_register(T_OBJECT);
-    __ oop2reg(method()->encoding(), meth);
+    __ oop2reg(method()->constant_encoding(), meth);
     args->append(meth);
     call_runtime(&signature, args, CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), voidType, NULL);
   }
@@ -2207,7 +2207,7 @@
     LIR_OprList* args = new LIR_OprList();
     args->append(getThreadPointer());
     LIR_Opr meth = new_register(T_OBJECT);
-    __ oop2reg(method()->encoding(), meth);
+    __ oop2reg(method()->constant_encoding(), meth);
     args->append(meth);
     call_runtime(&signature, args, CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), voidType, NULL);
   }
@@ -2216,7 +2216,7 @@
     LIR_Opr obj;
     if (method()->is_static()) {
       obj = new_register(T_OBJECT);
-      __ oop2reg(method()->holder()->java_mirror()->encoding(), obj);
+      __ oop2reg(method()->holder()->java_mirror()->constant_encoding(), obj);
     } else {
       Local* receiver = x->state()->local_at(0)->as_Local();
       assert(receiver != NULL, "must already exist");
@@ -2660,7 +2660,7 @@
     }
 
     LIR_Opr meth = new_register(T_OBJECT);
-    __ oop2reg(method()->encoding(), meth);
+    __ oop2reg(method()->constant_encoding(), meth);
     LIR_Opr result = increment_and_return_counter(meth, offset, InvocationCounter::count_increment);
     __ cmp(lir_cond_aboveEqual, result, LIR_OprFact::intConst(limit));
     CodeStub* overflow = new CounterOverflowStub(info, info->bci());

--- a/src/share/vm/c1/c1_ValueType.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/c1/c1_ValueType.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -86,7 +86,7 @@
 
 jobject ObjectType::encoding() const {
   assert(is_constant(), "must be");
-  return constant_value()->encoding();
+  return constant_value()->constant_encoding();
 }
 
 bool ObjectType::is_loaded() const {

--- a/src/share/vm/ci/ciEnv.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/ci/ciEnv.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -257,7 +257,7 @@
 
 // ------------------------------------------------------------------
 // ciEnv::make_array
-ciArray* ciEnv::make_array(GrowableArray<ciObject*>* objects) {
+ciArray* ciEnv::make_system_array(GrowableArray<ciObject*>* objects) {
   VM_ENTRY_MARK;
   int length = objects->length();
   objArrayOop a = oopFactory::new_system_objArray(length, THREAD);

--- a/src/share/vm/ci/ciEnv.hpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/ci/ciEnv.hpp	Wed Sep 30 13:25:39 2009 -0700
@@ -339,8 +339,8 @@
   // but consider adding to vmSymbols.hpp instead.
 
   // Use this to make a holder for non-perm compile time constants.
-  // The resulting array is guaranteed to satisfy "has_encoding".
-  ciArray*  make_array(GrowableArray<ciObject*>* objects);
+  // The resulting array is guaranteed to satisfy "can_be_constant".
+  ciArray*  make_system_array(GrowableArray<ciObject*>* objects);
 
   // converts the ciKlass* representing the holder of a method into a
   // ciInstanceKlass*.  This is needed since the holder of a method in

--- a/src/share/vm/ci/ciMethod.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/ci/ciMethod.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -325,10 +325,10 @@
 }
 
 // ------------------------------------------------------------------
-// ciMethod::liveness_at_bci
+// ciMethod::raw_liveness_at_bci
 //
 // Which local variables are live at a specific bci?
-MethodLivenessResult ciMethod::liveness_at_bci(int bci) {
+MethodLivenessResult ciMethod::raw_liveness_at_bci(int bci) {
   check_is_loaded();
   if (_liveness == NULL) {
     // Create the liveness analyzer.
@@ -336,7 +336,17 @@
     _liveness = new (arena) MethodLiveness(arena, this);
     _liveness->compute_liveness();
   }
-  MethodLivenessResult result = _liveness->get_liveness_at(bci);
+  return _liveness->get_liveness_at(bci);
+}
+
+// ------------------------------------------------------------------
+// ciMethod::liveness_at_bci
+//
+// Which local variables are live at a specific bci?  When debugging
+// will return true for all locals in some cases to improve debug
+// information.
+MethodLivenessResult ciMethod::liveness_at_bci(int bci) {
+  MethodLivenessResult result = raw_liveness_at_bci(bci);
   if (CURRENT_ENV->jvmti_can_access_local_variables() || DeoptimizeALot || CompileTheWorld) {
     // Keep all locals live for the user's edification and amusement.
     result.at_put_range(0, result.size(), true);

--- a/src/share/vm/ci/ciMethod.hpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/ci/ciMethod.hpp	Wed Sep 30 13:25:39 2009 -0700
@@ -149,6 +149,12 @@
   bool          has_monitor_bytecodes() const    { return _uses_monitors; }
   bool          has_balanced_monitors();
 
+  // Returns a bitmap indicating which locals are required to be
+  // maintained as live for deopt.  raw_liveness_at_bci is always the
+  // direct output of the liveness computation while liveness_at_bci
+  // may mark all locals as live to improve support for debugging Java
+  // code by maintaining the state of as many locals as possible.
+  MethodLivenessResult raw_liveness_at_bci(int bci);
   MethodLivenessResult liveness_at_bci(int bci);
 
   // Get the interpreters viewpoint on oop liveness.  MethodLiveness is

--- a/src/share/vm/ci/ciObject.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/ci/ciObject.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -55,6 +55,7 @@
   }
   _klass = NULL;
   _ident = 0;
+  init_flags_from(o);
 }
 
 // ------------------------------------------------------------------
@@ -69,6 +70,7 @@
   }
   _klass = NULL;
   _ident = 0;
+  init_flags_from(h());
 }
 
 // ------------------------------------------------------------------
@@ -158,7 +160,7 @@
 }
 
 // ------------------------------------------------------------------
-// ciObject::encoding
+// ciObject::constant_encoding
 //
 // The address which the compiler should embed into the
 // generated code to represent this oop.  This address
@@ -172,16 +174,24 @@
 //
 // This method should be changed to return an generified address
 // to discourage use of the JNI handle.
-jobject ciObject::encoding() {
+jobject ciObject::constant_encoding() {
   assert(is_null_object() || handle() != NULL, "cannot embed null pointer");
-  assert(has_encoding(), "oop must be NULL or perm");
+  assert(can_be_constant(), "oop must be NULL or perm");
   return handle();
 }
 
 // ------------------------------------------------------------------
-// ciObject::has_encoding
-bool ciObject::has_encoding() {
-  return handle() == NULL || is_perm();
+// ciObject::can_be_constant
+bool ciObject::can_be_constant() {
+  if (ScavengeRootsInCode >= 1)  return true;  // now everybody can encode as a constant
+  return handle() == NULL || !is_scavengable();
+}
+
+// ------------------------------------------------------------------
+// ciObject::should_be_constant()
+bool ciObject::should_be_constant() {
+  if (ScavengeRootsInCode >= 2)  return true;  // force everybody to be a constant
+  return handle() == NULL || !is_scavengable();
 }
 
 
@@ -195,8 +205,9 @@
 void ciObject::print(outputStream* st) {
   st->print("<%s", type_string());
   GUARDED_VM_ENTRY(print_impl(st);)
-  st->print(" ident=%d %s address=0x%x>", ident(),
+  st->print(" ident=%d %s%s address=0x%x>", ident(),
         is_perm() ? "PERM" : "",
+        is_scavengable() ? "SCAVENGABLE" : "",
         (address)this);
 }
 

--- a/src/share/vm/ci/ciObject.hpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/ci/ciObject.hpp	Wed Sep 30 13:25:39 2009 -0700
@@ -51,9 +51,10 @@
   ciKlass* _klass;
   uint     _ident;
 
-  enum { FLAG_BITS   = 1};
+  enum { FLAG_BITS   = 2 };
   enum {
-         PERM_FLAG    = 1
+         PERM_FLAG        = 1,
+         SCAVENGABLE_FLAG = 2
        };
 protected:
   ciObject();
@@ -68,8 +69,15 @@
     return JNIHandles::resolve_non_null(_handle);
   }
 
-  void set_perm() {
-    _ident |=  PERM_FLAG;
+  void init_flags_from(oop x) {
+    int flags = 0;
+    if (x != NULL) {
+      if (x->is_perm())
+        flags |= PERM_FLAG;
+      if (x->is_scavengable())
+        flags |= SCAVENGABLE_FLAG;
+    }
+    _ident |= flags;
   }
 
   // Virtual behavior of the print() method.
@@ -91,17 +99,27 @@
   // A hash value for the convenience of compilers.
   int hash();
 
-  // Tells if this oop has an encoding.  (I.e., is it null or perm?)
+  // Tells if this oop has an encoding as a constant.
+  // True if is_scavengable is false.
+  // Also true if ScavengeRootsInCode is non-zero.
   // If it does not have an encoding, the compiler is responsible for
   // making other arrangements for dealing with the oop.
-  // See ciEnv::make_perm_array
-  bool has_encoding();
+  // See ciEnv::make_array
+  bool can_be_constant();
+
+  // Tells if this oop should be made a constant.
+  // True if is_scavengable is false or ScavengeRootsInCode > 1.
+  bool should_be_constant();
 
   // Is this object guaranteed to be in the permanent part of the heap?
   // If so, CollectedHeap::can_elide_permanent_oop_store_barriers is relevant.
   // If the answer is false, no guarantees are made.
   bool is_perm() { return (_ident & PERM_FLAG) != 0; }
 
+  // Might this object possibly move during a scavenge operation?
+  // If the answer is true and ScavengeRootsInCode==0, the oop cannot be embedded in code.
+  bool is_scavengable() { return (_ident & SCAVENGABLE_FLAG) != 0; }
+
   // The address which the compiler should embed into the
   // generated code to represent this oop.  This address
   // is not the true address of the oop -- it will get patched
@@ -109,7 +127,7 @@
   //
   // Usage note: no address arithmetic allowed.  Oop must
   // be registered with the oopRecorder.
-  jobject encoding();
+  jobject constant_encoding();
 
   // What kind of ciObject is this?
   virtual bool is_null_object() const       { return false; }

--- a/src/share/vm/ci/ciObjectFactory.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/ci/ciObjectFactory.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -261,12 +261,11 @@
     ciObject* new_object = create_new_object(keyHandle());
     assert(keyHandle() == new_object->get_oop(), "must be properly recorded");
     init_ident_of(new_object);
-    if (!keyHandle->is_perm()) {
+    if (!new_object->is_perm()) {
       // Not a perm-space object.
       insert_non_perm(bucket, keyHandle(), new_object);
       return new_object;
     }
-    new_object->set_perm();
     if (len != _ci_objects->length()) {
       // creating the new object has recursively entered new objects
       // into the table.  We need to recompute our index.

--- a/src/share/vm/ci/ciTypeFlow.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/ci/ciTypeFlow.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -2486,8 +2486,13 @@
         // Assume irreducible entries need more data flow
         add_to_work_list(succ);
       }
-      lp = lp->parent();
-      assert(lp != NULL, "nested loop must have parent by now");
+      Loop* plp = lp->parent();
+      if (plp == NULL) {
+        // This only happens for some irreducible cases.  The parent
+        // will be updated during a later pass.
+        break;
+      }
+      lp = plp;
     }
 
     // Merge loop tree branch for all successors.

--- a/src/share/vm/classfile/systemDictionary.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/classfile/systemDictionary.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -2417,6 +2417,8 @@
                          vmSymbols::makeSite_name(), vmSymbols::makeSite_signature(),
                          &args, CHECK_(empty));
   oop call_site_oop = (oop) result.get_jobject();
+  assert(call_site_oop->is_oop()
+         /*&& sun_dyn_CallSiteImpl::is_instance(call_site_oop)*/, "must be sane");
   sun_dyn_CallSiteImpl::set_vmmethod(call_site_oop, mh_invdyn());
   if (TraceMethodHandles) {
     tty->print_cr("Linked invokedynamic bci=%d site="INTPTR_FORMAT":", caller_bci, call_site_oop);
@@ -2453,6 +2455,8 @@
   oop boot_method_oop = (oop) result.get_jobject();
 
   if (boot_method_oop != NULL) {
+    assert(boot_method_oop->is_oop()
+           && java_dyn_MethodHandle::is_instance(boot_method_oop), "must be sane");
     // probably no race conditions, but let's be careful:
     if (Atomic::cmpxchg_ptr(boot_method_oop, ik->adr_bootstrap_method(), NULL) == NULL)
       ik->set_bootstrap_method(boot_method_oop);

--- a/src/share/vm/code/codeBlob.hpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/code/codeBlob.hpp	Wed Sep 30 13:25:39 2009 -0700
@@ -175,6 +175,8 @@
                             OopClosure* keep_alive,
                             bool unloading_occurred);
   virtual void oops_do(OopClosure* f) = 0;
+  // (All CodeBlob subtypes other than NMethod currently have
+  // an empty oops_do() method.
 
   // OopMap for frame
   OopMapSet* oop_maps() const                    { return _oop_maps; }

--- a/src/share/vm/code/codeCache.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/code/codeCache.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -95,6 +95,7 @@
 int CodeCache::_number_of_blobs = 0;
 int CodeCache::_number_of_nmethods_with_dependencies = 0;
 bool CodeCache::_needs_cache_clean = false;
+nmethod* CodeCache::_scavenge_root_nmethods = NULL;
 
 
 CodeBlob* CodeCache::first() {
@@ -148,10 +149,7 @@
     }
   }
   verify_if_often();
-  if (PrintCodeCache2) {        // Need to add a new flag
-      ResourceMark rm;
-      tty->print_cr("CodeCache allocation:  addr: " INTPTR_FORMAT ", size: 0x%x\n", cb, size);
-  }
+  print_trace("allocation", cb, size);
   return cb;
 }
 
@@ -159,10 +157,7 @@
   assert_locked_or_safepoint(CodeCache_lock);
   verify_if_often();
 
-  if (PrintCodeCache2) {        // Need to add a new flag
-      ResourceMark rm;
-      tty->print_cr("CodeCache free:  addr: " INTPTR_FORMAT ", size: 0x%x\n", cb, cb->size());
-  }
+  print_trace("free", cb);
   if (cb->is_nmethod() && ((nmethod *)cb)->has_dependencies()) {
     _number_of_nmethods_with_dependencies--;
   }
@@ -260,14 +255,148 @@
   }
 }
 
-void CodeCache::oops_do(OopClosure* f) {
+void CodeCache::blobs_do(CodeBlobClosure* f) {
   assert_locked_or_safepoint(CodeCache_lock);
   FOR_ALL_ALIVE_BLOBS(cb) {
-    cb->oops_do(f);
+    f->do_code_blob(cb);
+
+#ifdef ASSERT
+    if (cb->is_nmethod())
+      ((nmethod*)cb)->verify_scavenge_root_oops();
+#endif //ASSERT
   }
 }
 
+// Walk the list of methods which might contain non-perm oops.
+void CodeCache::scavenge_root_nmethods_do(CodeBlobClosure* f) {
+  assert_locked_or_safepoint(CodeCache_lock);
+  debug_only(mark_scavenge_root_nmethods());
+
+  for (nmethod* cur = scavenge_root_nmethods(); cur != NULL; cur = cur->scavenge_root_link()) {
+    debug_only(cur->clear_scavenge_root_marked());
+    assert(cur->scavenge_root_not_marked(), "");
+    assert(cur->on_scavenge_root_list(), "else shouldn't be on this list");
+
+    bool is_live = (!cur->is_zombie() && !cur->is_unloaded());
+#ifndef PRODUCT
+    if (TraceScavenge) {
+      cur->print_on(tty, is_live ? "scavenge root" : "dead scavenge root"); tty->cr();
+    }
+#endif //PRODUCT
+    if (is_live)
+      // Perform cur->oops_do(f), maybe just once per nmethod.
+      f->do_code_blob(cur);
+  }
+
+  // Check for stray marks.
+  debug_only(verify_perm_nmethods(NULL));
+}
+
+void CodeCache::add_scavenge_root_nmethod(nmethod* nm) {
+  assert_locked_or_safepoint(CodeCache_lock);
+  nm->set_on_scavenge_root_list();
+  nm->set_scavenge_root_link(_scavenge_root_nmethods);
+  set_scavenge_root_nmethods(nm);
+  print_trace("add_scavenge_root", nm);
+}
+
+void CodeCache::drop_scavenge_root_nmethod(nmethod* nm) {
+  assert_locked_or_safepoint(CodeCache_lock);
+  print_trace("drop_scavenge_root", nm);
+  nmethod* last = NULL;
+  nmethod* cur = scavenge_root_nmethods();
+  while (cur != NULL) {
+    nmethod* next = cur->scavenge_root_link();
+    if (cur == nm) {
+      if (last != NULL)
+            last->set_scavenge_root_link(next);
+      else  set_scavenge_root_nmethods(next);
+      nm->set_scavenge_root_link(NULL);
+      nm->clear_on_scavenge_root_list();
+      return;
+    }
+    last = cur;
+    cur = next;
+  }
+  assert(false, "should have been on list");
+}
+
+void CodeCache::prune_scavenge_root_nmethods() {
+  assert_locked_or_safepoint(CodeCache_lock);
+  debug_only(mark_scavenge_root_nmethods());
+
+  nmethod* last = NULL;
+  nmethod* cur = scavenge_root_nmethods();
+  while (cur != NULL) {
+    nmethod* next = cur->scavenge_root_link();
+    debug_only(cur->clear_scavenge_root_marked());
+    assert(cur->scavenge_root_not_marked(), "");
+    assert(cur->on_scavenge_root_list(), "else shouldn't be on this list");
+
+    if (!cur->is_zombie() && !cur->is_unloaded()
+        && cur->detect_scavenge_root_oops()) {
+      // Keep it.  Advance 'last' to prevent deletion.
+      last = cur;
+    } else {
+      // Prune it from the list, so we don't have to look at it any more.
+      print_trace("prune_scavenge_root", cur);
+      cur->set_scavenge_root_link(NULL);
+      cur->clear_on_scavenge_root_list();
+      if (last != NULL)
+            last->set_scavenge_root_link(next);
+      else  set_scavenge_root_nmethods(next);
+    }
+    cur = next;
+  }
+
+  // Check for stray marks.
+  debug_only(verify_perm_nmethods(NULL));
+}
+
+#ifndef PRODUCT
+void CodeCache::asserted_non_scavengable_nmethods_do(CodeBlobClosure* f) {
+  // While we are here, verify the integrity of the list.
+  mark_scavenge_root_nmethods();
+  for (nmethod* cur = scavenge_root_nmethods(); cur != NULL; cur = cur->scavenge_root_link()) {
+    assert(cur->on_scavenge_root_list(), "else shouldn't be on this list");
+    cur->clear_scavenge_root_marked();
+  }
+  verify_perm_nmethods(f);
+}
+
+// Temporarily mark nmethods that are claimed to be on the non-perm list.
+void CodeCache::mark_scavenge_root_nmethods() {
+  FOR_ALL_ALIVE_BLOBS(cb) {
+    if (cb->is_nmethod()) {
+      nmethod *nm = (nmethod*)cb;
+      assert(nm->scavenge_root_not_marked(), "clean state");
+      if (nm->on_scavenge_root_list())
+        nm->set_scavenge_root_marked();
+    }
+  }
+}
+
+// If the closure is given, run it on the unlisted nmethods.
+// Also make sure that the effects of mark_scavenge_root_nmethods is gone.
+void CodeCache::verify_perm_nmethods(CodeBlobClosure* f_or_null) {
+  FOR_ALL_ALIVE_BLOBS(cb) {
+    bool call_f = (f_or_null != NULL);
+    if (cb->is_nmethod()) {
+      nmethod *nm = (nmethod*)cb;
+      assert(nm->scavenge_root_not_marked(), "must be already processed");
+      if (nm->on_scavenge_root_list())
+        call_f = false;  // don't show this one to the client
+      nm->verify_scavenge_root_oops();
+    } else {
+      call_f = false;   // not an nmethod
+    }
+    if (call_f)  f_or_null->do_code_blob(cb);
+  }
+}
+#endif //PRODUCT
+
 void CodeCache::gc_prologue() {
+  assert(!nmethod::oops_do_marking_is_active(), "oops_do_marking_epilogue must be called");
 }
 
 
@@ -285,6 +414,8 @@
     cb->fix_oop_relocations();
   }
   set_needs_cache_clean(false);
+  prune_scavenge_root_nmethods();
+  assert(!nmethod::oops_do_marking_is_active(), "oops_do_marking_prologue must be called");
 }
 
 
@@ -508,6 +639,14 @@
   }
 }
 
+void CodeCache::print_trace(const char* event, CodeBlob* cb, int size) {
+  if (PrintCodeCache2) {  // Need to add a new flag
+    ResourceMark rm;
+    if (size == 0)  size = cb->size();
+    tty->print_cr("CodeCache %s:  addr: " INTPTR_FORMAT ", size: 0x%x", event, cb, size);
+  }
+}
+
 void CodeCache::print_internals() {
   int nmethodCount = 0;
   int runtimeStubCount = 0;

--- a/src/share/vm/code/codeCache.hpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/code/codeCache.hpp	Wed Sep 30 13:25:39 2009 -0700
@@ -45,8 +45,13 @@
   static int _number_of_blobs;
   static int _number_of_nmethods_with_dependencies;
   static bool _needs_cache_clean;
+  static nmethod* _scavenge_root_nmethods;  // linked via nm->scavenge_root_link()
 
   static void verify_if_often() PRODUCT_RETURN;
+
+  static void mark_scavenge_root_nmethods() PRODUCT_RETURN;
+  static void verify_perm_nmethods(CodeBlobClosure* f_or_null) PRODUCT_RETURN;
+
  public:
 
   // Initialization
@@ -61,6 +66,7 @@
   static void flush();                              // flushes all CodeBlobs
   static bool contains(void *p);                    // returns whether p is included
   static void blobs_do(void f(CodeBlob* cb));       // iterates over all CodeBlobs
+  static void blobs_do(CodeBlobClosure* f);         // iterates over all CodeBlobs
   static void nmethods_do(void f(nmethod* nm));     // iterates over all nmethods
 
   // Lookup
@@ -106,12 +112,24 @@
   static void do_unloading(BoolObjectClosure* is_alive,
                            OopClosure* keep_alive,
                            bool unloading_occurred);
-  static void oops_do(OopClosure* f);
+  static void oops_do(OopClosure* f) {
+    CodeBlobToOopClosure oopc(f, /*do_marking=*/ false);
+    blobs_do(&oopc);
+  }
+  static void asserted_non_scavengable_nmethods_do(CodeBlobClosure* f = NULL) PRODUCT_RETURN;
+  static void scavenge_root_nmethods_do(CodeBlobClosure* f);
+
+  static nmethod* scavenge_root_nmethods()          { return _scavenge_root_nmethods; }
+  static void set_scavenge_root_nmethods(nmethod* nm) { _scavenge_root_nmethods = nm; }
+  static void add_scavenge_root_nmethod(nmethod* nm);
+  static void drop_scavenge_root_nmethod(nmethod* nm);
+  static void prune_scavenge_root_nmethods();
 
   // Printing/debugging
   static void print()   PRODUCT_RETURN;          // prints summary
   static void print_internals();
   static void verify();                          // verifies the code cache
+  static void print_trace(const char* event, CodeBlob* cb, int size = 0) PRODUCT_RETURN;
 
   // The full limits of the codeCache
   static address  low_bound()                    { return (address) _heap->low_boundary(); }

--- a/src/share/vm/code/debugInfoRec.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/code/debugInfoRec.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -299,7 +299,7 @@
   stream()->write_int(sender_stream_offset);
 
   // serialize scope
-  jobject method_enc = (method == NULL)? NULL: method->encoding();
+  jobject method_enc = (method == NULL)? NULL: method->constant_encoding();
   stream()->write_int(oop_recorder()->find_index(method_enc));
   stream()->write_bci(bci);
   assert(method == NULL ||

--- a/src/share/vm/code/dependencies.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/code/dependencies.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -302,7 +302,7 @@
       bytes.write_byte(code_byte);
       for (int j = 0; j < stride; j++) {
         if (j == skipj)  continue;
-        bytes.write_int(_oop_recorder->find_index(deps->at(i+j)->encoding()));
+        bytes.write_int(_oop_recorder->find_index(deps->at(i+j)->constant_encoding()));
       }
     }
   }

--- a/src/share/vm/code/nmethod.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/code/nmethod.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -581,10 +581,13 @@
     debug_only(No_Safepoint_Verifier nsv;)
     assert_locked_or_safepoint(CodeCache_lock);
 
-    NOT_PRODUCT(_has_debug_info = false; )
+    NOT_PRODUCT(_has_debug_info = false);
+    _oops_do_mark_link       = NULL;
     _method                  = method;
     _entry_bci               = InvocationEntryBci;
-    _link                    = NULL;
+    _osr_link                = NULL;
+    _scavenge_root_link      = NULL;
+    _scavenge_root_state     = 0;
     _compiler                = NULL;
     // We have no exception handler or deopt handler make the
     // values something that will never match a pc like the nmethod vtable entry
@@ -618,7 +621,7 @@
     _stack_traversal_mark    = 0;
 
     code_buffer->copy_oops_to(this);
-    debug_only(check_store();)
+    debug_only(verify_scavenge_root_oops());
     CodeCache::commit(this);
     VTune::create_nmethod(this);
   }
@@ -668,10 +671,13 @@
     debug_only(No_Safepoint_Verifier nsv;)
     assert_locked_or_safepoint(CodeCache_lock);
 
-    NOT_PRODUCT(_has_debug_info = false; )
+    NOT_PRODUCT(_has_debug_info = false);
+    _oops_do_mark_link       = NULL;
     _method                  = method;
     _entry_bci               = InvocationEntryBci;
-    _link                    = NULL;
+    _osr_link                = NULL;
+    _scavenge_root_link      = NULL;
+    _scavenge_root_state     = 0;
     _compiler                = NULL;
     // We have no exception handler or deopt handler make the
     // values something that will never match a pc like the nmethod vtable entry
@@ -703,7 +709,7 @@
     _stack_traversal_mark    = 0;
 
     code_buffer->copy_oops_to(this);
-    debug_only(check_store();)
+    debug_only(verify_scavenge_root_oops());
     CodeCache::commit(this);
     VTune::create_nmethod(this);
   }
@@ -770,12 +776,15 @@
     debug_only(No_Safepoint_Verifier nsv;)
     assert_locked_or_safepoint(CodeCache_lock);
 
-    NOT_PRODUCT(_has_debug_info = false; )
+    NOT_PRODUCT(_has_debug_info = false);
+    _oops_do_mark_link       = NULL;
     _method                  = method;
     _compile_id              = compile_id;
     _comp_level              = comp_level;
     _entry_bci               = entry_bci;
-    _link                    = NULL;
+    _osr_link                = NULL;
+    _scavenge_root_link      = NULL;
+    _scavenge_root_state     = 0;
     _compiler                = compiler;
     _orig_pc_offset          = orig_pc_offset;
 #ifdef HAVE_DTRACE_H
@@ -813,7 +822,10 @@
     code_buffer->copy_oops_to(this);
     debug_info->copy_to(this);
     dependencies->copy_to(this);
-    debug_only(check_store();)
+    if (ScavengeRootsInCode && detect_scavenge_root_oops()) {
+      CodeCache::add_scavenge_root_nmethod(this);
+    }
+    debug_only(verify_scavenge_root_oops());
 
     CodeCache::commit(this);
 
@@ -902,23 +914,30 @@
   if (st != NULL) {
     ttyLocker ttyl;
     // Print a little tag line that looks like +PrintCompilation output:
-    st->print("%3d%c  %s",
+    int tlen = (int) strlen(title);
+    bool do_nl = false;
+    if (tlen > 0 && title[tlen-1] == '\n') { tlen--; do_nl = true; }
+    st->print("%3d%c  %.*s",
               compile_id(),
               is_osr_method() ? '%' :
               method() != NULL &&
               is_native_method() ? 'n' : ' ',
-              title);
+              tlen, title);
 #ifdef TIERED
     st->print(" (%d) ", comp_level());
 #endif // TIERED
     if (WizardMode) st->print(" (" INTPTR_FORMAT ")", this);
-    if (method() != NULL) {
-      method()->print_short_name(st);
+    if (Universe::heap()->is_gc_active() && method() != NULL) {
+      st->print("(method)");
+    } else if (method() != NULL) {
+        method()->print_short_name(st);
       if (is_osr_method())
         st->print(" @ %d", osr_entry_bci());
       if (method()->code_size() > 0)
         st->print(" (%d bytes)", method()->code_size());
     }
+
+    if (do_nl)  st->cr();
   }
 }
 
@@ -1033,6 +1052,7 @@
   }
 }
 
+// This is a private interface with the sweeper.
 void nmethod::mark_as_seen_on_stack() {
   assert(is_not_entrant(), "must be a non-entrant method");
   set_stack_traversal_mark(NMethodSweeper::traversal_count());
@@ -1077,7 +1097,8 @@
                   " unloadable], methodOop(" INTPTR_FORMAT
                   "), cause(" INTPTR_FORMAT ")",
                   this, (address)_method, (address)cause);
-    cause->klass()->print();
+    if (!Universe::heap()->is_gc_active())
+      cause->klass()->print();
   }
   // Unlink the osr method, so we do not look this up again
   if (is_osr_method()) {
@@ -1109,7 +1130,8 @@
   // The methodOop is gone at this point
   assert(_method == NULL, "Tautology");
 
-  set_link(NULL);
+  set_osr_link(NULL);
+  //set_scavenge_root_link(NULL); // done by prune_scavenge_root_nmethods
   NMethodSweeper::notify(this);
 }
 
@@ -1295,6 +1317,10 @@
     ec = next;
   }
 
+  if (on_scavenge_root_list()) {
+    CodeCache::drop_scavenge_root_nmethod(this);
+  }
+
   ((CodeBlob*)(this))->flush();
 
   CodeCache::free(this);
@@ -1354,7 +1380,10 @@
       return false;
     }
   }
-  assert(unloading_occurred, "Inconsistency in unloading");
+  // If ScavengeRootsInCode is true, an nmethod might be unloaded
+  // simply because one of its constant oops has gone dead.
+  // No actual classes need to be unloaded in order for this to occur.
+  assert(unloading_occurred || ScavengeRootsInCode, "Inconsistency in unloading");
   make_unloaded(is_alive, obj);
   return true;
 }
@@ -1529,13 +1558,12 @@
 // the (strong) marking phase, and then again when walking
 // the code cache contents during the weak roots processing
 // phase. The two uses are distinguished by means of the
-// do_nmethods() method in the closure "f" below -- which
-// answers "yes" in the first case, and "no" in the second
+// 'do_strong_roots_only' flag, which is true in the first
 // case. We want to walk the weak roots in the nmethod
 // only in the second case. The weak roots in the nmethod
 // are the oops in the ExceptionCache and the InlineCache
 // oops.
-void nmethod::oops_do(OopClosure* f) {
+void nmethod::oops_do(OopClosure* f, bool do_strong_roots_only) {
   // make sure the oops ready to receive visitors
   assert(!is_zombie() && !is_unloaded(),
          "should not call follow on zombie or unloaded nmethod");
@@ -1553,7 +1581,7 @@
 
   // Compiled code
   f->do_oop((oop*) &_method);
-  if (!f->do_nmethods()) {
+  if (!do_strong_roots_only) {
     // weak roots processing phase -- update ExceptionCache oops
     ExceptionCache* ec = exception_cache();
     while(ec != NULL) {
@@ -1579,12 +1607,108 @@
   }
 
   // Scopes
+  // This includes oop constants not inlined in the code stream.
   for (oop* p = oops_begin(); p < oops_end(); p++) {
     if (*p == Universe::non_oop_word())  continue;  // skip non-oops
     f->do_oop(p);
   }
 }
 
+#define NMETHOD_SENTINEL ((nmethod*)badAddress)
+
+nmethod* volatile nmethod::_oops_do_mark_nmethods;
+
+// An nmethod is "marked" if its _mark_link is set non-null.
+// Even if it is the end of the linked list, it will have a non-null link value,
+// as long as it is on the list.
+// This code must be MP safe, because it is used from parallel GC passes.
+bool nmethod::test_set_oops_do_mark() {
+  assert(nmethod::oops_do_marking_is_active(), "oops_do_marking_prologue must be called");
+  nmethod* observed_mark_link = _oops_do_mark_link;
+  if (observed_mark_link == NULL) {
+    // Claim this nmethod for this thread to mark.
+    observed_mark_link = (nmethod*)
+      Atomic::cmpxchg_ptr(NMETHOD_SENTINEL, &_oops_do_mark_link, NULL);
+    if (observed_mark_link == NULL) {
+
+      // Atomically append this nmethod (now claimed) to the head of the list:
+      nmethod* observed_mark_nmethods = _oops_do_mark_nmethods;
+      for (;;) {
+        nmethod* required_mark_nmethods = observed_mark_nmethods;
+        _oops_do_mark_link = required_mark_nmethods;
+        observed_mark_nmethods = (nmethod*)
+          Atomic::cmpxchg_ptr(this, &_oops_do_mark_nmethods, required_mark_nmethods);
+        if (observed_mark_nmethods == required_mark_nmethods)
+          break;
+      }
+      // Mark was clear when we first saw this guy.
+      NOT_PRODUCT(if (TraceScavenge)  print_on(tty, "oops_do, mark\n"));
+      return false;
+    }
+  }
+  // On fall through, another racing thread marked this nmethod before we did.
+  return true;
+}
+
+void nmethod::oops_do_marking_prologue() {
+  NOT_PRODUCT(if (TraceScavenge)  tty->print_cr("[oops_do_marking_prologue"));
+  assert(_oops_do_mark_nmethods == NULL, "must not call oops_do_marking_prologue twice in a row");
+  // We use cmpxchg_ptr instead of regular assignment here because the user
+  // may fork a bunch of threads, and we need them all to see the same state.
+  void* observed = Atomic::cmpxchg_ptr(NMETHOD_SENTINEL, &_oops_do_mark_nmethods, NULL);
+  guarantee(observed == NULL, "no races in this sequential code");
+}
+
+void nmethod::oops_do_marking_epilogue() {
+  assert(_oops_do_mark_nmethods != NULL, "must not call oops_do_marking_epilogue twice in a row");
+  nmethod* cur = _oops_do_mark_nmethods;
+  while (cur != NMETHOD_SENTINEL) {
+    assert(cur != NULL, "not NULL-terminated");
+    nmethod* next = cur->_oops_do_mark_link;
+    cur->_oops_do_mark_link = NULL;
+    NOT_PRODUCT(if (TraceScavenge)  cur->print_on(tty, "oops_do, unmark\n"));
+    cur = next;
+  }
+  void* required = _oops_do_mark_nmethods;
+  void* observed = Atomic::cmpxchg_ptr(NULL, &_oops_do_mark_nmethods, required);
+  guarantee(observed == required, "no races in this sequential code");
+  NOT_PRODUCT(if (TraceScavenge)  tty->print_cr("oops_do_marking_epilogue]"));
+}
+
+class DetectScavengeRoot: public OopClosure {
+  bool     _detected_scavenge_root;
+public:
+  DetectScavengeRoot() : _detected_scavenge_root(false)
+  { NOT_PRODUCT(_print_nm = NULL); }
+  bool detected_scavenge_root() { return _detected_scavenge_root; }
+  virtual void do_oop(oop* p) {
+    if ((*p) != NULL && (*p)->is_scavengable()) {
+      NOT_PRODUCT(maybe_print(p));
+      _detected_scavenge_root = true;
+    }
+  }
+  virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); }
+
+#ifndef PRODUCT
+  nmethod* _print_nm;
+  void maybe_print(oop* p) {
+    if (_print_nm == NULL)  return;
+    if (!_detected_scavenge_root)  _print_nm->print_on(tty, "new scavenge root");
+    tty->print_cr(""PTR_FORMAT"[offset=%d] detected non-perm oop "PTR_FORMAT" (found at "PTR_FORMAT")",
+                  _print_nm, (int)((intptr_t)p - (intptr_t)_print_nm),
+                  (intptr_t)(*p), (intptr_t)p);
+    (*p)->print();
+  }
+#endif //PRODUCT
+};
+
+bool nmethod::detect_scavenge_root_oops() {
+  DetectScavengeRoot detect_scavenge_root;
+  NOT_PRODUCT(if (TraceScavenge)  detect_scavenge_root._print_nm = this);
+  oops_do(&detect_scavenge_root);
+  return detect_scavenge_root.detected_scavenge_root();
+}
+
 // Method that knows how to preserve outgoing arguments at call. This method must be
 // called with a frame corresponding to a Java invoke
 void nmethod::preserve_callee_argument_oops(frame fr, const RegisterMap *reg_map, OopClosure* f) {
@@ -1899,6 +2023,24 @@
 // -----------------------------------------------------------------------------
 // Verification
 
+class VerifyOopsClosure: public OopClosure {
+  nmethod* _nm;
+  bool     _ok;
+public:
+  VerifyOopsClosure(nmethod* nm) : _nm(nm), _ok(true) { }
+  bool ok() { return _ok; }
+  virtual void do_oop(oop* p) {
+    if ((*p) == NULL || (*p)->is_oop())  return;
+    if (_ok) {
+      _nm->print_nmethod(true);
+      _ok = false;
+    }
+    tty->print_cr("*** non-oop "PTR_FORMAT" found at "PTR_FORMAT" (offset %d)",
+                  (intptr_t)(*p), (intptr_t)p, (int)((intptr_t)p - (intptr_t)_nm));
+  }
+  virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); }
+};
+
 void nmethod::verify() {
 
   // Hmm. OSR methods can be deopted but not marked as zombie or not_entrant
@@ -1932,6 +2074,11 @@
     }
   }
 
+  VerifyOopsClosure voc(this);
+  oops_do(&voc);
+  assert(voc.ok(), "embedded oops must be OK");
+  verify_scavenge_root_oops();
+
   verify_scopes();
 }
 
@@ -1995,19 +2142,34 @@
 // Non-product code
 #ifndef PRODUCT
 
-void nmethod::check_store() {
-  // Make sure all oops in the compiled code are tenured
+class DebugScavengeRoot: public OopClosure {
+  nmethod* _nm;
+  bool     _ok;
+public:
+  DebugScavengeRoot(nmethod* nm) : _nm(nm), _ok(true) { }
+  bool ok() { return _ok; }
+  virtual void do_oop(oop* p) {
+    if ((*p) == NULL || !(*p)->is_scavengable())  return;
+    if (_ok) {
+      _nm->print_nmethod(true);
+      _ok = false;
+    }
+    tty->print_cr("*** non-perm oop "PTR_FORMAT" found at "PTR_FORMAT" (offset %d)",
+                  (intptr_t)(*p), (intptr_t)p, (int)((intptr_t)p - (intptr_t)_nm));
+    (*p)->print();
+  }
+  virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); }
+};
 
-  RelocIterator iter(this);
-  while (iter.next()) {
-    if (iter.type() == relocInfo::oop_type) {
-      oop_Relocation* reloc = iter.oop_reloc();
-      oop obj = reloc->oop_value();
-      if (obj != NULL && !obj->is_perm()) {
-        fatal("must be permanent oop in compiled code");
-      }
-    }
+void nmethod::verify_scavenge_root_oops() {
+  if (!on_scavenge_root_list()) {
+    // Actually look inside, to verify the claim that it's clean.
+    DebugScavengeRoot debug_scavenge_root(this);
+    oops_do(&debug_scavenge_root);
+    if (!debug_scavenge_root.ok())
+      fatal("found an unadvertised bad non-perm oop in the code cache");
   }
+  assert(scavenge_root_not_marked(), "");
 }
 
 #endif // PRODUCT
@@ -2040,6 +2202,7 @@
     if (is_not_entrant()) tty->print("not_entrant ");
     if (is_zombie())      tty->print("zombie ");
     if (is_unloaded())    tty->print("unloaded ");
+    if (on_scavenge_root_list())  tty->print("scavenge_root ");
     tty->print_cr("}:");
   }
   if (size              () > 0) tty->print_cr(" total in heap  [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d",

--- a/src/share/vm/code/nmethod.hpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/code/nmethod.hpp	Wed Sep 30 13:25:39 2009 -0700
@@ -125,6 +125,7 @@
 class nmethod : public CodeBlob {
   friend class VMStructs;
   friend class NMethodSweeper;
+  friend class CodeCache;  // non-perm oops
  private:
   // Shared fields for all nmethod's
   static int _zombie_instruction_size;
@@ -132,7 +133,12 @@
   methodOop _method;
   int       _entry_bci;        // != InvocationEntryBci if this nmethod is an on-stack replacement method
 
-  nmethod*  _link;             // To support simple linked-list chaining of nmethods
+  // To support simple linked-list chaining of nmethods:
+  nmethod*  _osr_link;         // from instanceKlass::osr_nmethods_head
+  nmethod*  _scavenge_root_link; // from CodeCache::scavenge_root_nmethods
+
+  static nmethod* volatile _oops_do_mark_nmethods;
+  nmethod*        volatile _oops_do_mark_link;
 
   AbstractCompiler* _compiler; // The compiler which compiled this nmethod
 
@@ -174,6 +180,8 @@
   // used by jvmti to track if an unload event has been posted for this nmethod.
   bool _unload_reported;
 
+  jbyte _scavenge_root_state;
+
   NOT_PRODUCT(bool _has_debug_info; )
 
   // Nmethod Flushing lock (if non-zero, then the nmethod is not removed)
@@ -242,7 +250,6 @@
 
   // helper methods
   void* operator new(size_t size, int nmethod_size);
-  void check_store();
 
   const char* reloc_string_for(u_char* begin, u_char* end);
   void make_not_entrant_or_zombie(int state);
@@ -406,6 +413,24 @@
   int   version() const                           { return flags.version; }
   void  set_version(int v);
 
+  // Non-perm oop support
+  bool  on_scavenge_root_list() const                  { return (_scavenge_root_state & 1) != 0; }
+ protected:
+  enum { npl_on_list = 0x01, npl_marked = 0x10 };
+  void  set_on_scavenge_root_list()                    { _scavenge_root_state = npl_on_list; }
+  void  clear_on_scavenge_root_list()                  { _scavenge_root_state = 0; }
+  // assertion-checking and pruning logic uses the bits of _scavenge_root_state
+#ifndef PRODUCT
+  void  set_scavenge_root_marked()                     { _scavenge_root_state |= npl_marked; }
+  void  clear_scavenge_root_marked()                   { _scavenge_root_state &= ~npl_marked; }
+  bool  scavenge_root_not_marked()                     { return (_scavenge_root_state &~ npl_on_list) == 0; }
+  // N.B. there is no positive marked query, and we only use the not_marked query for asserts.
+#endif //PRODUCT
+  nmethod* scavenge_root_link() const                  { return _scavenge_root_link; }
+  void     set_scavenge_root_link(nmethod *n)          { _scavenge_root_link = n; }
+
+ public:
+
   // Sweeper support
   long  stack_traversal_mark()                    { return _stack_traversal_mark; }
   void  set_stack_traversal_mark(long l)          { _stack_traversal_mark = l; }
@@ -424,8 +449,8 @@
   int   osr_entry_bci() const                     { assert(_entry_bci != InvocationEntryBci, "wrong kind of nmethod"); return _entry_bci; }
   address  osr_entry() const                      { assert(_entry_bci != InvocationEntryBci, "wrong kind of nmethod"); return _osr_entry_point; }
   void  invalidate_osr_method();
-  nmethod* link() const                           { return _link; }
-  void     set_link(nmethod *n)                   { _link = n; }
+  nmethod* osr_link() const                       { return _osr_link; }
+  void     set_osr_link(nmethod *n)               { _osr_link = n; }
 
   // tells whether frames described by this nmethod can be deoptimized
   // note: native wrappers cannot be deoptimized.
@@ -465,7 +490,16 @@
 
   void preserve_callee_argument_oops(frame fr, const RegisterMap *reg_map,
                                      OopClosure* f);
-  void oops_do(OopClosure* f);
+  virtual void oops_do(OopClosure* f) { oops_do(f, false); }
+  void         oops_do(OopClosure* f, bool do_strong_roots_only);
+  bool detect_scavenge_root_oops();
+  void verify_scavenge_root_oops() PRODUCT_RETURN;
+
+  bool test_set_oops_do_mark();
+  static void oops_do_marking_prologue();
+  static void oops_do_marking_epilogue();
+  static bool oops_do_marking_is_active() { return _oops_do_mark_nmethods != NULL; }
+  DEBUG_ONLY(bool test_oops_do_mark() { return _oops_do_mark_link != NULL; })
 
   // ScopeDesc for an instruction
   ScopeDesc* scope_desc_at(address pc);

--- a/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp	Wed Sep 30 13:25:39 2009 -0700
@@ -47,20 +47,15 @@
  private:
   const MemRegion _span;
   CMSBitMap*      _bitMap;
-  const bool      _should_do_nmethods;
  protected:
   DO_OOP_WORK_DEFN
  public:
-  MarkRefsIntoClosure(MemRegion span, CMSBitMap* bitMap,
-                      bool should_do_nmethods);
+  MarkRefsIntoClosure(MemRegion span, CMSBitMap* bitMap);
   virtual void do_oop(oop* p);
   virtual void do_oop(narrowOop* p);
   inline void do_oop_nv(oop* p)       { MarkRefsIntoClosure::do_oop_work(p); }
   inline void do_oop_nv(narrowOop* p) { MarkRefsIntoClosure::do_oop_work(p); }
   bool do_header() { return true; }
-  virtual const bool do_nmethods() const {
-    return _should_do_nmethods;
-  }
   Prefetch::style prefetch_style() {
     return Prefetch::do_read;
   }
@@ -73,20 +68,16 @@
   const MemRegion _span;
   CMSBitMap*      _verification_bm;
   CMSBitMap*      _cms_bm;
-  const bool      _should_do_nmethods;
  protected:
   DO_OOP_WORK_DEFN
  public:
   MarkRefsIntoVerifyClosure(MemRegion span, CMSBitMap* verification_bm,
-                            CMSBitMap* cms_bm, bool should_do_nmethods);
+                            CMSBitMap* cms_bm);
   virtual void do_oop(oop* p);
   virtual void do_oop(narrowOop* p);
   inline void do_oop_nv(oop* p)       { MarkRefsIntoVerifyClosure::do_oop_work(p); }
   inline void do_oop_nv(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
   bool do_header() { return true; }
-  virtual const bool do_nmethods() const {
-    return _should_do_nmethods;
-  }
   Prefetch::style prefetch_style() {
     return Prefetch::do_read;
   }
@@ -228,7 +219,6 @@
   inline void do_oop_nv(oop* p)       { MarkRefsIntoAndScanClosure::do_oop_work(p); }
   inline void do_oop_nv(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
   bool do_header() { return true; }
-  virtual const bool do_nmethods() const { return true; }
   Prefetch::style prefetch_style() {
     return Prefetch::do_read;
   }
@@ -273,7 +263,6 @@
   inline void do_oop_nv(oop* p)       { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
   inline void do_oop_nv(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
   bool do_header() { return true; }
-  virtual const bool do_nmethods() const { return true; }
   // When ScanMarkedObjectsAgainClosure is used,
   // it passes [Par_]MarkRefsIntoAndScanClosure to oop_oop_iterate(),
   // and this delegation is used.

--- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp	Fri Sep 25 04:39:54 2009 -0700
+++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp	Wed Sep 30 13:25:39 2009 -0700
@@ -2852,14 +2852,17 @@
   GenCollectedHeap* gch = GenCollectedHeap::heap();
 
   // Mark from roots one level into CMS
-  MarkRefsIntoClosure notOlder(_span, verification_mark_bm(), true /* nmethods */);
+  MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
   gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
 
   gch->gen_process_strong_roots(_cmsGen->level(),
                                 true,   // younger gens are roots
+                                true,   // activate StrongRootsScope
                                 true,   // collecting perm gen
                                 SharedHeap::ScanningOption(roots_scanning_options()),
-                                NULL, &notOlder);
+                                &notOlder,
+                                true,   // walk code active on stacks
+                                NULL);
 
   // Now mark from the roots
   assert(_revisitStack.isEmpty(), "Should be empty");
@@ -2901,13 +2904,16 @@
 
   // Mark from roots one level into CMS
   MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(),
-                                     markBitMap(), true /* nmethods */);
+                                     markBitMap());
   gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
   gch->gen_process_strong_roots(_cmsGen->level(),
                                 true,   // younger gens are roots
+                                true,   // activate StrongRootsScope
                                 true,   // collecting perm gen
                                 SharedHeap::ScanningOption(roots_scanning_options()),
-                                NULL, &notOlder);
+                                &notOlder,
+                                true,   // walk code active on stacks
+                                NULL);
 
   // Now mark from the roots
   assert(_revisitStack.isEmpty(), "Should be empty");
@@ -3484,8 +3490,10 @@
   FalseClosure falseClosure;
   // In the case of a synchronous collection, we will elide the
   // remark step, so it's important to catch all the nmethod oops
-  // in this step; hence the last argument to the constrcutor below.
-  MarkRefsIntoClosure notOlder(_span, &_markBitMap, !asynch /* nmethods */);
+  // in this step.
+  // The final 'true' flag to gen_process_strong_roots will ensure this.
+  // If 'async' is true, we can relax the nmethod tracing.
+  MarkRefsIntoClosure notOlder(_span, &_markBitMap);
   GenCollectedHeap* gch = GenCollectedHeap::heap();
 
   verify_work_stacks_empty();
@@ -3504,9 +3512,12 @@
     gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
     gch->gen_process_strong_roots(_cmsGen->level(),
                                   true,   // younger gens are roots
+                                  true,   // activate StrongRootsScope
                                   true,   // collecting perm gen
                                   SharedHeap::ScanningOption(roots_scanning_options()),
-                                  NULL, &notOlder);
+                                  &notOlder,
+                                  true,   // walk all of code cache if (so & SO_CodeCache)
+                                  NULL);
   }
 
   // Clear mod-union table; it will be dirtied in the prologue of
@@ -5040,9 +5051,15 @@
   _timer.start();
   gch->gen_process_strong_roots(_collector->_cmsGen->level(),
                                 false,     // yg was scanned above
+                                false,     // this is parallel code
                                 true,      // collecting perm gen
                                 SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
-                                NULL, &par_mrias_cl);
+                                &par_mrias_cl,
+                                true,   // walk all of code cache if (so & SO_CodeCache)
+                                NULL);
+  assert(_collector->should_unload_classes()
+         || (_collector->CMSCollector::roots_scanning_options() & SharedHeap::SO_CodeCache),
+         "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
   _timer.stop();
   if (PrintCMSStatistics != 0) {
     gclog_or_tty->print_cr(
@@ -5423,7 +5440,6 @@
 
   // Set up for parallel process_strong_roots work.
   gch->set_par_threads(n_workers);
-  gch->change_strong_roots_parity();
   // We won't be iterating over the cards in the card table updating
   // the younger_gen cards, so we shouldn't call the following else
   // the verification code as well as subsequent younger_refs_iterate
@@ -5454,8 +5470,10 @@
   if (n_workers > 1) {
     // Make refs discovery MT-safe
     ReferenceProcessorMTMutator mt(ref_processor(), true);
+    GenCollectedHeap::StrongRootsScope srs(gch);
     workers->run_task(&tsk);
   } else {
+    GenCollectedHeap::StrongRootsScope srs(gch);
     tsk.work(0);
   }
   gch->set_par_threads(0);  // 0 ==> non-parallel.
@@ -5539,11 +5557,18 @@
     verify_work_stacks_empty();
 
     gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
+    GenCollectedHeap::StrongRootsScope srs(gch);
     gch->gen_process_strong_roots(_cmsGen->level(),
                                   true,  // younger gens as roots
+                                  false, // use the local StrongRootsScope
                                   true,  // collecting perm gen
                                   SharedHeap::ScanningOption(roots_scanning_options()),
-                                  NULL, &mrias_cl);
+                                  &mrias_cl,
+                                  true,   // walk code active on stacks
+                                  NULL);
+    assert(should_unload_classes()
+           || (roots_scanning_options() & SharedHeap::SO_CodeCache),
+           "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
   }
   verify_work_stacks_empty();
   // Restore evacuated mark words, if any, used for overflow list links
@@ -6418,10 +6443,9 @@
 // generation then this will lose younger_gen cards!
 
 MarkRefsIntoClosure::MarkRefsIntoClosure(
-  MemRegion span, CMSBitMap* bitMap, bool should_do_nmethods):
+  MemRegion span, CMSBitMap* bitMap):
     _span(span),
-    _bitMap(bitMap),
-    _should_do_nmethods(should_do_nmethods)
+    _bitMap(bitMap)
 {
     assert(_ref_processor == NULL, "deliberately left NULL");
     assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
@@ -6442,12 +6466,11 @@
 
 // A variant of the above, used for CMS marking verification.
 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
-  MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm,
-  bool should_do_nmethods):
+  MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
     _span(span),
     _verification_bm(verification_bm),
-    _cms_bm(cms_bm),
-    _should_do_nmethods(should_do_nmethods) {
+    _cms_bm(cms_bm)
+{
     assert(_ref_processor == NULL, "deliberately left NULL");
     assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
 }