Mercurial > hg > truffle
view src/share/vm/gc_implementation/concurrentMarkSweep/binaryTreeDictionary.cpp @ 5343:20c14eb46238
added GraalOptions.CheckcastMinHintHitProbability to better guide use of hints for checkcasts
author | Doug Simon <doug.simon@oracle.com> |
---|---|
date | Wed, 02 May 2012 17:09:00 +0200 |
parents | f95d63e2154a |
children |
line wrap: on
line source
/* * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. * */ #include "precompiled.hpp" #include "gc_implementation/concurrentMarkSweep/binaryTreeDictionary.hpp" #include "gc_implementation/shared/allocationStats.hpp" #include "gc_implementation/shared/spaceDecorator.hpp" #include "memory/space.inline.hpp" #include "runtime/globals.hpp" #include "utilities/ostream.hpp" //////////////////////////////////////////////////////////////////////////////// // A binary tree based search structure for free blocks. // This is currently used in the Concurrent Mark&Sweep implementation. //////////////////////////////////////////////////////////////////////////////// TreeChunk* TreeChunk::as_TreeChunk(FreeChunk* fc) { // Do some assertion checking here. return (TreeChunk*) fc; } void TreeChunk::verifyTreeChunkList() const { TreeChunk* nextTC = (TreeChunk*)next(); if (prev() != NULL) { // interior list node shouldn'r have tree fields guarantee(embedded_list()->parent() == NULL && embedded_list()->left() == NULL && embedded_list()->right() == NULL, "should be clear"); } if (nextTC != NULL) { guarantee(as_TreeChunk(nextTC->prev()) == this, "broken chain"); guarantee(nextTC->size() == size(), "wrong size"); nextTC->verifyTreeChunkList(); } } TreeList* TreeList::as_TreeList(TreeChunk* tc) { // This first free chunk in the list will be the tree list. assert(tc->size() >= sizeof(TreeChunk), "Chunk is too small for a TreeChunk"); TreeList* tl = tc->embedded_list(); tc->set_list(tl); #ifdef ASSERT tl->set_protecting_lock(NULL); #endif tl->set_hint(0); tl->set_size(tc->size()); tl->link_head(tc); tl->link_tail(tc); tl->set_count(1); tl->init_statistics(true /* split_birth */); tl->setParent(NULL); tl->setLeft(NULL); tl->setRight(NULL); return tl; } TreeList* TreeList::as_TreeList(HeapWord* addr, size_t size) { TreeChunk* tc = (TreeChunk*) addr; assert(size >= sizeof(TreeChunk), "Chunk is too small for a TreeChunk"); // The space in the heap will have been mangled initially but // is not remangled when a free chunk is returned to the free list // (since it is used to maintain the chunk on the free list). assert((ZapUnusedHeapArea && SpaceMangler::is_mangled((HeapWord*) tc->size_addr()) && SpaceMangler::is_mangled((HeapWord*) tc->prev_addr()) && SpaceMangler::is_mangled((HeapWord*) tc->next_addr())) || (tc->size() == 0 && tc->prev() == NULL && tc->next() == NULL), "Space should be clear or mangled"); tc->setSize(size); tc->linkPrev(NULL); tc->linkNext(NULL); TreeList* tl = TreeList::as_TreeList(tc); return tl; } TreeList* TreeList::removeChunkReplaceIfNeeded(TreeChunk* tc) { TreeList* retTL = this; FreeChunk* list = head(); assert(!list || list != list->next(), "Chunk on list twice"); assert(tc != NULL, "Chunk being removed is NULL"); assert(parent() == NULL || this == parent()->left() || this == parent()->right(), "list is inconsistent"); assert(tc->isFree(), "Header is not marked correctly"); assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); FreeChunk* prevFC = tc->prev(); TreeChunk* nextTC = TreeChunk::as_TreeChunk(tc->next()); assert(list != NULL, "should have at least the target chunk"); // Is this the first item on the list? if (tc == list) { // The "getChunk..." functions for a TreeList will not return the // first chunk in the list unless it is the last chunk in the list // because the first chunk is also acting as the tree node. // When coalescing happens, however, the first chunk in the a tree // list can be the start of a free range. Free ranges are removed // from the free lists so that they are not available to be // allocated when the sweeper yields (giving up the free list lock) // to allow mutator activity. If this chunk is the first in the // list and is not the last in the list, do the work to copy the // TreeList from the first chunk to the next chunk and update all // the TreeList pointers in the chunks in the list. if (nextTC == NULL) { assert(prevFC == NULL, "Not last chunk in the list"); set_tail(NULL); set_head(NULL); } else { // copy embedded list. nextTC->set_embedded_list(tc->embedded_list()); retTL = nextTC->embedded_list(); // Fix the pointer to the list in each chunk in the list. // This can be slow for a long list. Consider having // an option that does not allow the first chunk on the // list to be coalesced. for (TreeChunk* curTC = nextTC; curTC != NULL; curTC = TreeChunk::as_TreeChunk(curTC->next())) { curTC->set_list(retTL); } // Fix the parent to point to the new TreeList. if (retTL->parent() != NULL) { if (this == retTL->parent()->left()) { retTL->parent()->setLeft(retTL); } else { assert(this == retTL->parent()->right(), "Parent is incorrect"); retTL->parent()->setRight(retTL); } } // Fix the children's parent pointers to point to the // new list. assert(right() == retTL->right(), "Should have been copied"); if (retTL->right() != NULL) { retTL->right()->setParent(retTL); } assert(left() == retTL->left(), "Should have been copied"); if (retTL->left() != NULL) { retTL->left()->setParent(retTL); } retTL->link_head(nextTC); assert(nextTC->isFree(), "Should be a free chunk"); } } else { if (nextTC == NULL) { // Removing chunk at tail of list link_tail(prevFC); } // Chunk is interior to the list prevFC->linkAfter(nextTC); } // Below this point the embeded TreeList being used for the // tree node may have changed. Don't use "this" // TreeList*. // chunk should still be a free chunk (bit set in _prev) assert(!retTL->head() || retTL->size() == retTL->head()->size(), "Wrong sized chunk in list"); debug_only( tc->linkPrev(NULL); tc->linkNext(NULL); tc->set_list(NULL); bool prev_found = false; bool next_found = false; for (FreeChunk* curFC = retTL->head(); curFC != NULL; curFC = curFC->next()) { assert(curFC != tc, "Chunk is still in list"); if (curFC == prevFC) { prev_found = true; } if (curFC == nextTC) { next_found = true; } } assert(prevFC == NULL || prev_found, "Chunk was lost from list"); assert(nextTC == NULL || next_found, "Chunk was lost from list"); assert(retTL->parent() == NULL || retTL == retTL->parent()->left() || retTL == retTL->parent()->right(), "list is inconsistent"); ) retTL->decrement_count(); assert(tc->isFree(), "Should still be a free chunk"); assert(retTL->head() == NULL || retTL->head()->prev() == NULL, "list invariant"); assert(retTL->tail() == NULL || retTL->tail()->next() == NULL, "list invariant"); return retTL; } void TreeList::returnChunkAtTail(TreeChunk* chunk) { assert(chunk != NULL, "returning NULL chunk"); assert(chunk->list() == this, "list should be set for chunk"); assert(tail() != NULL, "The tree list is embedded in the first chunk"); // which means that the list can never be empty. assert(!verifyChunkInFreeLists(chunk), "Double entry"); assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); FreeChunk* fc = tail(); fc->linkAfter(chunk); link_tail(chunk); assert(!tail() || size() == tail()->size(), "Wrong sized chunk in list"); increment_count(); debug_only(increment_returnedBytes_by(chunk->size()*sizeof(HeapWord));) assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); } // Add this chunk at the head of the list. "At the head of the list" // is defined to be after the chunk pointer to by head(). This is // because the TreeList is embedded in the first TreeChunk in the // list. See the definition of TreeChunk. void TreeList::returnChunkAtHead(TreeChunk* chunk) { assert(chunk->list() == this, "list should be set for chunk"); assert(head() != NULL, "The tree list is embedded in the first chunk"); assert(chunk != NULL, "returning NULL chunk"); assert(!verifyChunkInFreeLists(chunk), "Double entry"); assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); FreeChunk* fc = head()->next(); if (fc != NULL) { chunk->linkAfter(fc); } else { assert(tail() == NULL, "List is inconsistent"); link_tail(chunk); } head()->linkAfter(chunk); assert(!head() || size() == head()->size(), "Wrong sized chunk in list"); increment_count(); debug_only(increment_returnedBytes_by(chunk->size()*sizeof(HeapWord));) assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); } TreeChunk* TreeList::head_as_TreeChunk() { assert(head() == NULL || TreeChunk::as_TreeChunk(head())->list() == this, "Wrong type of chunk?"); return TreeChunk::as_TreeChunk(head()); } TreeChunk* TreeList::first_available() { assert(head() != NULL, "The head of the list cannot be NULL"); FreeChunk* fc = head()->next(); TreeChunk* retTC; if (fc == NULL) { retTC = head_as_TreeChunk(); } else { retTC = TreeChunk::as_TreeChunk(fc); } assert(retTC->list() == this, "Wrong type of chunk."); return retTC; } // Returns the block with the largest heap address amongst // those in the list for this size; potentially slow and expensive, // use with caution! TreeChunk* TreeList::largest_address() { assert(head() != NULL, "The head of the list cannot be NULL"); FreeChunk* fc = head()->next(); TreeChunk* retTC; if (fc == NULL) { retTC = head_as_TreeChunk(); } else { // walk down the list and return the one with the highest // heap address among chunks of this size. FreeChunk* last = fc; while (fc->next() != NULL) { if ((HeapWord*)last < (HeapWord*)fc) { last = fc; } fc = fc->next(); } retTC = TreeChunk::as_TreeChunk(last); } assert(retTC->list() == this, "Wrong type of chunk."); return retTC; } BinaryTreeDictionary::BinaryTreeDictionary(MemRegion mr, bool splay): _splay(splay) { assert(mr.byte_size() > MIN_TREE_CHUNK_SIZE, "minimum chunk size"); reset(mr); assert(root()->left() == NULL, "reset check failed"); assert(root()->right() == NULL, "reset check failed"); assert(root()->head()->next() == NULL, "reset check failed"); assert(root()->head()->prev() == NULL, "reset check failed"); assert(totalSize() == root()->size(), "reset check failed"); assert(totalFreeBlocks() == 1, "reset check failed"); } void BinaryTreeDictionary::inc_totalSize(size_t inc) { _totalSize = _totalSize + inc; } void BinaryTreeDictionary::dec_totalSize(size_t dec) { _totalSize = _totalSize - dec; } void BinaryTreeDictionary::reset(MemRegion mr) { assert(mr.byte_size() > MIN_TREE_CHUNK_SIZE, "minimum chunk size"); set_root(TreeList::as_TreeList(mr.start(), mr.word_size())); set_totalSize(mr.word_size()); set_totalFreeBlocks(1); } void BinaryTreeDictionary::reset(HeapWord* addr, size_t byte_size) { MemRegion mr(addr, heap_word_size(byte_size)); reset(mr); } void BinaryTreeDictionary::reset() { set_root(NULL); set_totalSize(0); set_totalFreeBlocks(0); } // Get a free block of size at least size from tree, or NULL. // If a splay step is requested, the removal algorithm (only) incorporates // a splay step as follows: // . the search proceeds down the tree looking for a possible // match. At the (closest) matching location, an appropriate splay step is applied // (zig, zig-zig or zig-zag). A chunk of the appropriate size is then returned // if available, and if it's the last chunk, the node is deleted. A deteleted // node is replaced in place by its tree successor. TreeChunk* BinaryTreeDictionary::getChunkFromTree(size_t size, Dither dither, bool splay) { TreeList *curTL, *prevTL; TreeChunk* retTC = NULL; assert(size >= MIN_TREE_CHUNK_SIZE, "minimum chunk size"); if (FLSVerifyDictionary) { verifyTree(); } // starting at the root, work downwards trying to find match. // Remember the last node of size too great or too small. for (prevTL = curTL = root(); curTL != NULL;) { if (curTL->size() == size) { // exact match break; } prevTL = curTL; if (curTL->size() < size) { // proceed to right sub-tree curTL = curTL->right(); } else { // proceed to left sub-tree assert(curTL->size() > size, "size inconsistency"); curTL = curTL->left(); } } if (curTL == NULL) { // couldn't find exact match // try and find the next larger size by walking back up the search path for (curTL = prevTL; curTL != NULL;) { if (curTL->size() >= size) break; else curTL = curTL->parent(); } assert(curTL == NULL || curTL->count() > 0, "An empty list should not be in the tree"); } if (curTL != NULL) { assert(curTL->size() >= size, "size inconsistency"); if (UseCMSAdaptiveFreeLists) { // A candidate chunk has been found. If it is already under // populated, get a chunk associated with the hint for this // chunk. if (curTL->surplus() <= 0) { /* Use the hint to find a size with a surplus, and reset the hint. */ TreeList* hintTL = curTL; while (hintTL->hint() != 0) { assert(hintTL->hint() == 0 || hintTL->hint() > hintTL->size(), "hint points in the wrong direction"); hintTL = findList(hintTL->hint()); assert(curTL != hintTL, "Infinite loop"); if (hintTL == NULL || hintTL == curTL /* Should not happen but protect against it */ ) { // No useful hint. Set the hint to NULL and go on. curTL->set_hint(0); break; } assert(hintTL->size() > size, "hint is inconsistent"); if (hintTL->surplus() > 0) { // The hint led to a list that has a surplus. Use it. // Set the hint for the candidate to an overpopulated // size. curTL->set_hint(hintTL->size()); // Change the candidate. curTL = hintTL; break; } // The evm code reset the hint of the candidate as // at an interim point. Why? Seems like this leaves // the hint pointing to a list that didn't work. // curTL->set_hint(hintTL->size()); } } } // don't waste time splaying if chunk's singleton if (splay && curTL->head()->next() != NULL) { semiSplayStep(curTL); } retTC = curTL->first_available(); assert((retTC != NULL) && (curTL->count() > 0), "A list in the binary tree should not be NULL"); assert(retTC->size() >= size, "A chunk of the wrong size was found"); removeChunkFromTree(retTC); assert(retTC->isFree(), "Header is not marked correctly"); } if (FLSVerifyDictionary) { verify(); } return retTC; } TreeList* BinaryTreeDictionary::findList(size_t size) const { TreeList* curTL; for (curTL = root(); curTL != NULL;) { if (curTL->size() == size) { // exact match break; } if (curTL->size() < size) { // proceed to right sub-tree curTL = curTL->right(); } else { // proceed to left sub-tree assert(curTL->size() > size, "size inconsistency"); curTL = curTL->left(); } } return curTL; } bool BinaryTreeDictionary::verifyChunkInFreeLists(FreeChunk* tc) const { size_t size = tc->size(); TreeList* tl = findList(size); if (tl == NULL) { return false; } else { return tl->verifyChunkInFreeLists(tc); } } FreeChunk* BinaryTreeDictionary::findLargestDict() const { TreeList *curTL = root(); if (curTL != NULL) { while(curTL->right() != NULL) curTL = curTL->right(); return curTL->largest_address(); } else { return NULL; } } // Remove the current chunk from the tree. If it is not the last // chunk in a list on a tree node, just unlink it. // If it is the last chunk in the list (the next link is NULL), // remove the node and repair the tree. TreeChunk* BinaryTreeDictionary::removeChunkFromTree(TreeChunk* tc) { assert(tc != NULL, "Should not call with a NULL chunk"); assert(tc->isFree(), "Header is not marked correctly"); TreeList *newTL, *parentTL; TreeChunk* retTC; TreeList* tl = tc->list(); debug_only( bool removing_only_chunk = false; if (tl == _root) { if ((_root->left() == NULL) && (_root->right() == NULL)) { if (_root->count() == 1) { assert(_root->head() == tc, "Should only be this one chunk"); removing_only_chunk = true; } } } ) assert(tl != NULL, "List should be set"); assert(tl->parent() == NULL || tl == tl->parent()->left() || tl == tl->parent()->right(), "list is inconsistent"); bool complicatedSplice = false; retTC = tc; // Removing this chunk can have the side effect of changing the node // (TreeList*) in the tree. If the node is the root, update it. TreeList* replacementTL = tl->removeChunkReplaceIfNeeded(tc); assert(tc->isFree(), "Chunk should still be free"); assert(replacementTL->parent() == NULL || replacementTL == replacementTL->parent()->left() || replacementTL == replacementTL->parent()->right(), "list is inconsistent"); if (tl == root()) { assert(replacementTL->parent() == NULL, "Incorrectly replacing root"); set_root(replacementTL); } debug_only( if (tl != replacementTL) { assert(replacementTL->head() != NULL, "If the tree list was replaced, it should not be a NULL list"); TreeList* rhl = replacementTL->head_as_TreeChunk()->list(); TreeList* rtl = TreeChunk::as_TreeChunk(replacementTL->tail())->list(); assert(rhl == replacementTL, "Broken head"); assert(rtl == replacementTL, "Broken tail"); assert(replacementTL->size() == tc->size(), "Broken size"); } ) // Does the tree need to be repaired? if (replacementTL->count() == 0) { assert(replacementTL->head() == NULL && replacementTL->tail() == NULL, "list count is incorrect"); // Find the replacement node for the (soon to be empty) node being removed. // if we have a single (or no) child, splice child in our stead if (replacementTL->left() == NULL) { // left is NULL so pick right. right may also be NULL. newTL = replacementTL->right(); debug_only(replacementTL->clearRight();) } else if (replacementTL->right() == NULL) { // right is NULL newTL = replacementTL->left(); debug_only(replacementTL->clearLeft();) } else { // we have both children, so, by patriarchal convention, // my replacement is least node in right sub-tree complicatedSplice = true; newTL = removeTreeMinimum(replacementTL->right()); assert(newTL != NULL && newTL->left() == NULL && newTL->right() == NULL, "sub-tree minimum exists"); } // newTL is the replacement for the (soon to be empty) node. // newTL may be NULL. // should verify; we just cleanly excised our replacement if (FLSVerifyDictionary) { verifyTree(); } // first make newTL my parent's child if ((parentTL = replacementTL->parent()) == NULL) { // newTL should be root assert(tl == root(), "Incorrectly replacing root"); set_root(newTL); if (newTL != NULL) { newTL->clearParent(); } } else if (parentTL->right() == replacementTL) { // replacementTL is a right child parentTL->setRight(newTL); } else { // replacementTL is a left child assert(parentTL->left() == replacementTL, "should be left child"); parentTL->setLeft(newTL); } debug_only(replacementTL->clearParent();) if (complicatedSplice) { // we need newTL to get replacementTL's // two children assert(newTL != NULL && newTL->left() == NULL && newTL->right() == NULL, "newTL should not have encumbrances from the past"); // we'd like to assert as below: // assert(replacementTL->left() != NULL && replacementTL->right() != NULL, // "else !complicatedSplice"); // ... however, the above assertion is too strong because we aren't // guaranteed that replacementTL->right() is still NULL. // Recall that we removed // the right sub-tree minimum from replacementTL. // That may well have been its right // child! So we'll just assert half of the above: assert(replacementTL->left() != NULL, "else !complicatedSplice"); newTL->setLeft(replacementTL->left()); newTL->setRight(replacementTL->right()); debug_only( replacementTL->clearRight(); replacementTL->clearLeft(); ) } assert(replacementTL->right() == NULL && replacementTL->left() == NULL && replacementTL->parent() == NULL, "delete without encumbrances"); } assert(totalSize() >= retTC->size(), "Incorrect total size"); dec_totalSize(retTC->size()); // size book-keeping assert(totalFreeBlocks() > 0, "Incorrect total count"); set_totalFreeBlocks(totalFreeBlocks() - 1); assert(retTC != NULL, "null chunk?"); assert(retTC->prev() == NULL && retTC->next() == NULL, "should return without encumbrances"); if (FLSVerifyDictionary) { verifyTree(); } assert(!removing_only_chunk || _root == NULL, "root should be NULL"); return TreeChunk::as_TreeChunk(retTC); } // Remove the leftmost node (lm) in the tree and return it. // If lm has a right child, link it to the left node of // the parent of lm. TreeList* BinaryTreeDictionary::removeTreeMinimum(TreeList* tl) { assert(tl != NULL && tl->parent() != NULL, "really need a proper sub-tree"); // locate the subtree minimum by walking down left branches TreeList* curTL = tl; for (; curTL->left() != NULL; curTL = curTL->left()); // obviously curTL now has at most one child, a right child if (curTL != root()) { // Should this test just be removed? TreeList* parentTL = curTL->parent(); if (parentTL->left() == curTL) { // curTL is a left child parentTL->setLeft(curTL->right()); } else { // If the list tl has no left child, then curTL may be // the right child of parentTL. assert(parentTL->right() == curTL, "should be a right child"); parentTL->setRight(curTL->right()); } } else { // The only use of this method would not pass the root of the // tree (as indicated by the assertion above that the tree list // has a parent) but the specification does not explicitly exclude the // passing of the root so accomodate it. set_root(NULL); } debug_only( curTL->clearParent(); // Test if this needs to be cleared curTL->clearRight(); // recall, above, left child is already null ) // we just excised a (non-root) node, we should still verify all tree invariants if (FLSVerifyDictionary) { verifyTree(); } return curTL; } // Based on a simplification of the algorithm by Sleator and Tarjan (JACM 1985). // The simplifications are the following: // . we splay only when we delete (not when we insert) // . we apply a single spay step per deletion/access // By doing such partial splaying, we reduce the amount of restructuring, // while getting a reasonably efficient search tree (we think). // [Measurements will be needed to (in)validate this expectation.] void BinaryTreeDictionary::semiSplayStep(TreeList* tc) { // apply a semi-splay step at the given node: // . if root, norting needs to be done // . if child of root, splay once // . else zig-zig or sig-zag depending on path from grandparent if (root() == tc) return; warning("*** Splaying not yet implemented; " "tree operations may be inefficient ***"); } void BinaryTreeDictionary::insertChunkInTree(FreeChunk* fc) { TreeList *curTL, *prevTL; size_t size = fc->size(); assert(size >= MIN_TREE_CHUNK_SIZE, "too small to be a TreeList"); if (FLSVerifyDictionary) { verifyTree(); } // XXX: do i need to clear the FreeChunk fields, let me do it just in case // Revisit this later fc->clearNext(); fc->linkPrev(NULL); // work down from the _root, looking for insertion point for (prevTL = curTL = root(); curTL != NULL;) { if (curTL->size() == size) // exact match break; prevTL = curTL; if (curTL->size() > size) { // follow left branch curTL = curTL->left(); } else { // follow right branch assert(curTL->size() < size, "size inconsistency"); curTL = curTL->right(); } } TreeChunk* tc = TreeChunk::as_TreeChunk(fc); // This chunk is being returned to the binary tree. Its embedded // TreeList should be unused at this point. tc->initialize(); if (curTL != NULL) { // exact match tc->set_list(curTL); curTL->returnChunkAtTail(tc); } else { // need a new node in tree tc->clearNext(); tc->linkPrev(NULL); TreeList* newTL = TreeList::as_TreeList(tc); assert(((TreeChunk*)tc)->list() == newTL, "List was not initialized correctly"); if (prevTL == NULL) { // we are the only tree node assert(root() == NULL, "control point invariant"); set_root(newTL); } else { // insert under prevTL ... if (prevTL->size() < size) { // am right child assert(prevTL->right() == NULL, "control point invariant"); prevTL->setRight(newTL); } else { // am left child assert(prevTL->size() > size && prevTL->left() == NULL, "cpt pt inv"); prevTL->setLeft(newTL); } } } assert(tc->list() != NULL, "Tree list should be set"); inc_totalSize(size); // Method 'totalSizeInTree' walks through the every block in the // tree, so it can cause significant performance loss if there are // many blocks in the tree assert(!FLSVerifyDictionary || totalSizeInTree(root()) == totalSize(), "_totalSize inconsistency"); set_totalFreeBlocks(totalFreeBlocks() + 1); if (FLSVerifyDictionary) { verifyTree(); } } size_t BinaryTreeDictionary::maxChunkSize() const { verify_par_locked(); TreeList* tc = root(); if (tc == NULL) return 0; for (; tc->right() != NULL; tc = tc->right()); return tc->size(); } size_t BinaryTreeDictionary::totalListLength(TreeList* tl) const { size_t res; res = tl->count(); #ifdef ASSERT size_t cnt; FreeChunk* tc = tl->head(); for (cnt = 0; tc != NULL; tc = tc->next(), cnt++); assert(res == cnt, "The count is not being maintained correctly"); #endif return res; } size_t BinaryTreeDictionary::totalSizeInTree(TreeList* tl) const { if (tl == NULL) return 0; return (tl->size() * totalListLength(tl)) + totalSizeInTree(tl->left()) + totalSizeInTree(tl->right()); } double BinaryTreeDictionary::sum_of_squared_block_sizes(TreeList* const tl) const { if (tl == NULL) { return 0.0; } double size = (double)(tl->size()); double curr = size * size * totalListLength(tl); curr += sum_of_squared_block_sizes(tl->left()); curr += sum_of_squared_block_sizes(tl->right()); return curr; } size_t BinaryTreeDictionary::totalFreeBlocksInTree(TreeList* tl) const { if (tl == NULL) return 0; return totalListLength(tl) + totalFreeBlocksInTree(tl->left()) + totalFreeBlocksInTree(tl->right()); } size_t BinaryTreeDictionary::numFreeBlocks() const { assert(totalFreeBlocksInTree(root()) == totalFreeBlocks(), "_totalFreeBlocks inconsistency"); return totalFreeBlocks(); } size_t BinaryTreeDictionary::treeHeightHelper(TreeList* tl) const { if (tl == NULL) return 0; return 1 + MAX2(treeHeightHelper(tl->left()), treeHeightHelper(tl->right())); } size_t BinaryTreeDictionary::treeHeight() const { return treeHeightHelper(root()); } size_t BinaryTreeDictionary::totalNodesHelper(TreeList* tl) const { if (tl == NULL) { return 0; } return 1 + totalNodesHelper(tl->left()) + totalNodesHelper(tl->right()); } size_t BinaryTreeDictionary::totalNodesInTree(TreeList* tl) const { return totalNodesHelper(root()); } void BinaryTreeDictionary::dictCensusUpdate(size_t size, bool split, bool birth){ TreeList* nd = findList(size); if (nd) { if (split) { if (birth) { nd->increment_splitBirths(); nd->increment_surplus(); } else { nd->increment_splitDeaths(); nd->decrement_surplus(); } } else { if (birth) { nd->increment_coalBirths(); nd->increment_surplus(); } else { nd->increment_coalDeaths(); nd->decrement_surplus(); } } } // A list for this size may not be found (nd == 0) if // This is a death where the appropriate list is now // empty and has been removed from the list. // This is a birth associated with a LinAB. The chunk // for the LinAB is not in the dictionary. } bool BinaryTreeDictionary::coalDictOverPopulated(size_t size) { if (FLSAlwaysCoalesceLarge) return true; TreeList* list_of_size = findList(size); // None of requested size implies overpopulated. return list_of_size == NULL || list_of_size->coalDesired() <= 0 || list_of_size->count() > list_of_size->coalDesired(); } // Closures for walking the binary tree. // do_list() walks the free list in a node applying the closure // to each free chunk in the list // do_tree() walks the nodes in the binary tree applying do_list() // to each list at each node. class TreeCensusClosure : public StackObj { protected: virtual void do_list(FreeList* fl) = 0; public: virtual void do_tree(TreeList* tl) = 0; }; class AscendTreeCensusClosure : public TreeCensusClosure { public: void do_tree(TreeList* tl) { if (tl != NULL) { do_tree(tl->left()); do_list(tl); do_tree(tl->right()); } } }; class DescendTreeCensusClosure : public TreeCensusClosure { public: void do_tree(TreeList* tl) { if (tl != NULL) { do_tree(tl->right()); do_list(tl); do_tree(tl->left()); } } }; // For each list in the tree, calculate the desired, desired // coalesce, count before sweep, and surplus before sweep. class BeginSweepClosure : public AscendTreeCensusClosure { double _percentage; float _inter_sweep_current; float _inter_sweep_estimate; float _intra_sweep_estimate; public: BeginSweepClosure(double p, float inter_sweep_current, float inter_sweep_estimate, float intra_sweep_estimate) : _percentage(p), _inter_sweep_current(inter_sweep_current), _inter_sweep_estimate(inter_sweep_estimate), _intra_sweep_estimate(intra_sweep_estimate) { } void do_list(FreeList* fl) { double coalSurplusPercent = _percentage; fl->compute_desired(_inter_sweep_current, _inter_sweep_estimate, _intra_sweep_estimate); fl->set_coalDesired((ssize_t)((double)fl->desired() * coalSurplusPercent)); fl->set_beforeSweep(fl->count()); fl->set_bfrSurp(fl->surplus()); } }; // Used to search the tree until a condition is met. // Similar to TreeCensusClosure but searches the // tree and returns promptly when found. class TreeSearchClosure : public StackObj { protected: virtual bool do_list(FreeList* fl) = 0; public: virtual bool do_tree(TreeList* tl) = 0; }; #if 0 // Don't need this yet but here for symmetry. class AscendTreeSearchClosure : public TreeSearchClosure { public: bool do_tree(TreeList* tl) { if (tl != NULL) { if (do_tree(tl->left())) return true; if (do_list(tl)) return true; if (do_tree(tl->right())) return true; } return false; } }; #endif class DescendTreeSearchClosure : public TreeSearchClosure { public: bool do_tree(TreeList* tl) { if (tl != NULL) { if (do_tree(tl->right())) return true; if (do_list(tl)) return true; if (do_tree(tl->left())) return true; } return false; } }; // Searches the tree for a chunk that ends at the // specified address. class EndTreeSearchClosure : public DescendTreeSearchClosure { HeapWord* _target; FreeChunk* _found; public: EndTreeSearchClosure(HeapWord* target) : _target(target), _found(NULL) {} bool do_list(FreeList* fl) { FreeChunk* item = fl->head(); while (item != NULL) { if (item->end() == _target) { _found = item; return true; } item = item->next(); } return false; } FreeChunk* found() { return _found; } }; FreeChunk* BinaryTreeDictionary::find_chunk_ends_at(HeapWord* target) const { EndTreeSearchClosure etsc(target); bool found_target = etsc.do_tree(root()); assert(found_target || etsc.found() == NULL, "Consistency check"); assert(!found_target || etsc.found() != NULL, "Consistency check"); return etsc.found(); } void BinaryTreeDictionary::beginSweepDictCensus(double coalSurplusPercent, float inter_sweep_current, float inter_sweep_estimate, float intra_sweep_estimate) { BeginSweepClosure bsc(coalSurplusPercent, inter_sweep_current, inter_sweep_estimate, intra_sweep_estimate); bsc.do_tree(root()); } // Closures and methods for calculating total bytes returned to the // free lists in the tree. NOT_PRODUCT( class InitializeDictReturnedBytesClosure : public AscendTreeCensusClosure { public: void do_list(FreeList* fl) { fl->set_returnedBytes(0); } }; void BinaryTreeDictionary::initializeDictReturnedBytes() { InitializeDictReturnedBytesClosure idrb; idrb.do_tree(root()); } class ReturnedBytesClosure : public AscendTreeCensusClosure { size_t _dictReturnedBytes; public: ReturnedBytesClosure() { _dictReturnedBytes = 0; } void do_list(FreeList* fl) { _dictReturnedBytes += fl->returnedBytes(); } size_t dictReturnedBytes() { return _dictReturnedBytes; } }; size_t BinaryTreeDictionary::sumDictReturnedBytes() { ReturnedBytesClosure rbc; rbc.do_tree(root()); return rbc.dictReturnedBytes(); } // Count the number of entries in the tree. class treeCountClosure : public DescendTreeCensusClosure { public: uint count; treeCountClosure(uint c) { count = c; } void do_list(FreeList* fl) { count++; } }; size_t BinaryTreeDictionary::totalCount() { treeCountClosure ctc(0); ctc.do_tree(root()); return ctc.count; } ) // Calculate surpluses for the lists in the tree. class setTreeSurplusClosure : public AscendTreeCensusClosure { double percentage; public: setTreeSurplusClosure(double v) { percentage = v; } void do_list(FreeList* fl) { double splitSurplusPercent = percentage; fl->set_surplus(fl->count() - (ssize_t)((double)fl->desired() * splitSurplusPercent)); } }; void BinaryTreeDictionary::setTreeSurplus(double splitSurplusPercent) { setTreeSurplusClosure sts(splitSurplusPercent); sts.do_tree(root()); } // Set hints for the lists in the tree. class setTreeHintsClosure : public DescendTreeCensusClosure { size_t hint; public: setTreeHintsClosure(size_t v) { hint = v; } void do_list(FreeList* fl) { fl->set_hint(hint); assert(fl->hint() == 0 || fl->hint() > fl->size(), "Current hint is inconsistent"); if (fl->surplus() > 0) { hint = fl->size(); } } }; void BinaryTreeDictionary::setTreeHints(void) { setTreeHintsClosure sth(0); sth.do_tree(root()); } // Save count before previous sweep and splits and coalesces. class clearTreeCensusClosure : public AscendTreeCensusClosure { void do_list(FreeList* fl) { fl->set_prevSweep(fl->count()); fl->set_coalBirths(0); fl->set_coalDeaths(0); fl->set_splitBirths(0); fl->set_splitDeaths(0); } }; void BinaryTreeDictionary::clearTreeCensus(void) { clearTreeCensusClosure ctc; ctc.do_tree(root()); } // Do reporting and post sweep clean up. void BinaryTreeDictionary::endSweepDictCensus(double splitSurplusPercent) { // Does walking the tree 3 times hurt? setTreeSurplus(splitSurplusPercent); setTreeHints(); if (PrintGC && Verbose) { reportStatistics(); } clearTreeCensus(); } // Print summary statistics void BinaryTreeDictionary::reportStatistics() const { verify_par_locked(); gclog_or_tty->print("Statistics for BinaryTreeDictionary:\n" "------------------------------------\n"); size_t totalSize = totalChunkSize(debug_only(NULL)); size_t freeBlocks = numFreeBlocks(); gclog_or_tty->print("Total Free Space: %d\n", totalSize); gclog_or_tty->print("Max Chunk Size: %d\n", maxChunkSize()); gclog_or_tty->print("Number of Blocks: %d\n", freeBlocks); if (freeBlocks > 0) { gclog_or_tty->print("Av. Block Size: %d\n", totalSize/freeBlocks); } gclog_or_tty->print("Tree Height: %d\n", treeHeight()); } // Print census information - counts, births, deaths, etc. // for each list in the tree. Also print some summary // information. class PrintTreeCensusClosure : public AscendTreeCensusClosure { int _print_line; size_t _totalFree; FreeList _total; public: PrintTreeCensusClosure() { _print_line = 0; _totalFree = 0; } FreeList* total() { return &_total; } size_t totalFree() { return _totalFree; } void do_list(FreeList* fl) { if (++_print_line >= 40) { FreeList::print_labels_on(gclog_or_tty, "size"); _print_line = 0; } fl->print_on(gclog_or_tty); _totalFree += fl->count() * fl->size() ; total()->set_count( total()->count() + fl->count() ); total()->set_bfrSurp( total()->bfrSurp() + fl->bfrSurp() ); total()->set_surplus( total()->splitDeaths() + fl->surplus() ); total()->set_desired( total()->desired() + fl->desired() ); total()->set_prevSweep( total()->prevSweep() + fl->prevSweep() ); total()->set_beforeSweep(total()->beforeSweep() + fl->beforeSweep()); total()->set_coalBirths( total()->coalBirths() + fl->coalBirths() ); total()->set_coalDeaths( total()->coalDeaths() + fl->coalDeaths() ); total()->set_splitBirths(total()->splitBirths() + fl->splitBirths()); total()->set_splitDeaths(total()->splitDeaths() + fl->splitDeaths()); } }; void BinaryTreeDictionary::printDictCensus(void) const { gclog_or_tty->print("\nBinaryTree\n"); FreeList::print_labels_on(gclog_or_tty, "size"); PrintTreeCensusClosure ptc; ptc.do_tree(root()); FreeList* total = ptc.total(); FreeList::print_labels_on(gclog_or_tty, " "); total->print_on(gclog_or_tty, "TOTAL\t"); gclog_or_tty->print( "totalFree(words): " SIZE_FORMAT_W(16) " growth: %8.5f deficit: %8.5f\n", ptc.totalFree(), (double)(total->splitBirths() + total->coalBirths() - total->splitDeaths() - total->coalDeaths()) /(total->prevSweep() != 0 ? (double)total->prevSweep() : 1.0), (double)(total->desired() - total->count()) /(total->desired() != 0 ? (double)total->desired() : 1.0)); } class PrintFreeListsClosure : public AscendTreeCensusClosure { outputStream* _st; int _print_line; public: PrintFreeListsClosure(outputStream* st) { _st = st; _print_line = 0; } void do_list(FreeList* fl) { if (++_print_line >= 40) { FreeList::print_labels_on(_st, "size"); _print_line = 0; } fl->print_on(gclog_or_tty); size_t sz = fl->size(); for (FreeChunk* fc = fl->head(); fc != NULL; fc = fc->next()) { _st->print_cr("\t[" PTR_FORMAT "," PTR_FORMAT ") %s", fc, (HeapWord*)fc + sz, fc->cantCoalesce() ? "\t CC" : ""); } } }; void BinaryTreeDictionary::print_free_lists(outputStream* st) const { FreeList::print_labels_on(st, "size"); PrintFreeListsClosure pflc(st); pflc.do_tree(root()); } // Verify the following tree invariants: // . _root has no parent // . parent and child point to each other // . each node's key correctly related to that of its child(ren) void BinaryTreeDictionary::verifyTree() const { guarantee(root() == NULL || totalFreeBlocks() == 0 || totalSize() != 0, "_totalSize should't be 0?"); guarantee(root() == NULL || root()->parent() == NULL, "_root shouldn't have parent"); verifyTreeHelper(root()); } size_t BinaryTreeDictionary::verifyPrevFreePtrs(TreeList* tl) { size_t ct = 0; for (FreeChunk* curFC = tl->head(); curFC != NULL; curFC = curFC->next()) { ct++; assert(curFC->prev() == NULL || curFC->prev()->isFree(), "Chunk should be free"); } return ct; } // Note: this helper is recursive rather than iterative, so use with // caution on very deep trees; and watch out for stack overflow errors; // In general, to be used only for debugging. void BinaryTreeDictionary::verifyTreeHelper(TreeList* tl) const { if (tl == NULL) return; guarantee(tl->size() != 0, "A list must has a size"); guarantee(tl->left() == NULL || tl->left()->parent() == tl, "parent<-/->left"); guarantee(tl->right() == NULL || tl->right()->parent() == tl, "parent<-/->right");; guarantee(tl->left() == NULL || tl->left()->size() < tl->size(), "parent !> left"); guarantee(tl->right() == NULL || tl->right()->size() > tl->size(), "parent !< left"); guarantee(tl->head() == NULL || tl->head()->isFree(), "!Free"); guarantee(tl->head() == NULL || tl->head_as_TreeChunk()->list() == tl, "list inconsistency"); guarantee(tl->count() > 0 || (tl->head() == NULL && tl->tail() == NULL), "list count is inconsistent"); guarantee(tl->count() > 1 || tl->head() == tl->tail(), "list is incorrectly constructed"); size_t count = verifyPrevFreePtrs(tl); guarantee(count == (size_t)tl->count(), "Node count is incorrect"); if (tl->head() != NULL) { tl->head_as_TreeChunk()->verifyTreeChunkList(); } verifyTreeHelper(tl->left()); verifyTreeHelper(tl->right()); } void BinaryTreeDictionary::verify() const { verifyTree(); guarantee(totalSize() == totalSizeInTree(root()), "Total Size inconsistency"); }