ADD_TEST_EQ(THREAD_LOCAL_END_OFFSET,
art::Thread::ThreadLocalEndOffset<POINTER_SIZE>().Int32Value())
// Offset of field Thread::tlsPtr_.thread_local_objects.
-#define THREAD_LOCAL_OBJECTS_OFFSET (THREAD_LOCAL_END_OFFSET + __SIZEOF_POINTER__)
+#define THREAD_LOCAL_OBJECTS_OFFSET (THREAD_LOCAL_END_OFFSET + 2 * __SIZEOF_POINTER__)
ADD_TEST_EQ(THREAD_LOCAL_OBJECTS_OFFSET,
art::Thread::ThreadLocalObjectsOffset<POINTER_SIZE>().Int32Value())
};
std::ostream& operator<<(std::ostream& os, const AllocatorType& rhs);
+inline constexpr bool IsTLABAllocator(AllocatorType allocator) {
+ return allocator == kAllocatorTypeTLAB || allocator == kAllocatorTypeRegionTLAB;
+}
+
} // namespace gc
} // namespace art
size_t bytes_allocated;
size_t usable_size;
size_t new_num_bytes_allocated = 0;
- if (allocator == kAllocatorTypeTLAB || allocator == kAllocatorTypeRegionTLAB) {
+ if (IsTLABAllocator(allocator)) {
byte_count = RoundUp(byte_count, space::BumpPointerSpace::kAlignment);
}
// If we have a thread local allocation we don't need to update bytes allocated.
- if ((allocator == kAllocatorTypeTLAB || allocator == kAllocatorTypeRegionTLAB) &&
- byte_count <= self->TlabSize()) {
+ if (IsTLABAllocator(allocator) && byte_count <= self->TlabSize()) {
obj = self->AllocTlab(byte_count);
DCHECK(obj != nullptr) << "AllocTlab can't fail";
obj->SetClass(klass);
static const char* kRegionSpaceName = "main space (region space)";
+// If true, we log all GCs in the both the foreground and background. Used for debugging.
+static constexpr bool kLogAllGCs = false;
+
+// How much we grow the TLAB if we can do it.
+static constexpr size_t kPartialTlabSize = 16 * KB;
+static constexpr bool kUsePartialTlabs = true;
+
#if defined(__LP64__) || !defined(ADDRESS_SANITIZER)
// 300 MB (0x12c00000) - (default non-moving space capacity).
static uint8_t* const kPreferredAllocSpaceBegin =
const std::vector<uint64_t>& pause_times = GetCurrentGcIteration()->GetPauseTimes();
// Print the GC if it is an explicit GC (e.g. Runtime.gc()) or a slow GC
// (mutator time blocked >= long_pause_log_threshold_).
- bool log_gc = gc_cause == kGcCauseExplicit;
+ bool log_gc = kLogAllGCs || gc_cause == kGcCauseExplicit;
if (!log_gc && CareAboutPauseTimes()) {
// GC for alloc pauses the allocating thread, so consider it as a pause.
log_gc = duration > long_gc_log_threshold_ ||
size_t* usable_size,
size_t* bytes_tl_bulk_allocated) {
const AllocatorType allocator_type = GetCurrentAllocator();
- if (allocator_type == kAllocatorTypeTLAB) {
+ if (kUsePartialTlabs && alloc_size <= self->TlabRemainingCapacity()) {
+ DCHECK_GT(alloc_size, self->TlabSize());
+ // There is enough space if we grow the TLAB. Lets do that. This increases the
+ // TLAB bytes.
+ const size_t min_expand_size = alloc_size - self->TlabSize();
+ const size_t expand_bytes = std::max(
+ min_expand_size,
+ std::min(self->TlabRemainingCapacity() - self->TlabSize(), kPartialTlabSize));
+ if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, expand_bytes, grow))) {
+ return nullptr;
+ }
+ *bytes_tl_bulk_allocated = expand_bytes;
+ self->ExpandTlab(expand_bytes);
+ DCHECK_LE(alloc_size, self->TlabSize());
+ } else if (allocator_type == kAllocatorTypeTLAB) {
DCHECK(bump_pointer_space_ != nullptr);
const size_t new_tlab_size = alloc_size + kDefaultTLABSize;
if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, new_tlab_size, grow))) {
if (LIKELY(!IsOutOfMemoryOnAllocation(allocator_type,
space::RegionSpace::kRegionSize,
grow))) {
+ const size_t new_tlab_size = kUsePartialTlabs
+ ? std::max(alloc_size, kPartialTlabSize)
+ : gc::space::RegionSpace::kRegionSize;
// Try to allocate a tlab.
- if (!region_space_->AllocNewTlab(self)) {
+ if (!region_space_->AllocNewTlab(self, new_tlab_size)) {
// Failed to allocate a tlab. Try non-tlab.
return region_space_->AllocNonvirtual<false>(alloc_size,
bytes_allocated,
usable_size,
bytes_tl_bulk_allocated);
}
- *bytes_tl_bulk_allocated = space::RegionSpace::kRegionSize;
+ *bytes_tl_bulk_allocated = new_tlab_size;
// Fall-through to using the TLAB below.
} else {
// Check OOME for a non-tlab allocation.
void BumpPointerSpace::RevokeThreadLocalBuffersLocked(Thread* thread) {
objects_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalObjectsAllocated());
bytes_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalBytesAllocated());
- thread->SetTlab(nullptr, nullptr);
+ thread->SetTlab(nullptr, nullptr, nullptr);
}
bool BumpPointerSpace::AllocNewTlab(Thread* self, size_t bytes) {
if (start == nullptr) {
return false;
}
- self->SetTlab(start, start + bytes);
+ self->SetTlab(start, start + bytes, start + bytes);
return true;
}
#define ART_RUNTIME_GC_SPACE_REGION_SPACE_INL_H_
#include "region_space.h"
+#include "thread-inl.h"
namespace art {
namespace gc {
return nullptr;
}
+inline size_t RegionSpace::Region::BytesAllocated() const {
+ if (IsLarge()) {
+ DCHECK_LT(begin_ + kRegionSize, Top());
+ return static_cast<size_t>(Top() - begin_);
+ } else if (IsLargeTail()) {
+ DCHECK_EQ(begin_, Top());
+ return 0;
+ } else {
+ DCHECK(IsAllocated()) << static_cast<uint>(state_);
+ DCHECK_LE(begin_, Top());
+ size_t bytes;
+ if (is_a_tlab_) {
+ bytes = thread_->GetThreadLocalBytesAllocated();
+ } else {
+ bytes = static_cast<size_t>(Top() - begin_);
+ }
+ DCHECK_LE(bytes, kRegionSize);
+ return bytes;
+ }
+}
+
+
} // namespace space
} // namespace gc
} // namespace art
r->objects_allocated_.FetchAndAddSequentiallyConsistent(1);
}
-bool RegionSpace::AllocNewTlab(Thread* self) {
+bool RegionSpace::AllocNewTlab(Thread* self, size_t min_bytes) {
MutexLock mu(self, region_lock_);
RevokeThreadLocalBuffersLocked(self);
// Retain sufficient free regions for full evacuation.
r->SetTop(r->End());
r->is_a_tlab_ = true;
r->thread_ = self;
- self->SetTlab(r->Begin(), r->End());
+ self->SetTlab(r->Begin(), r->Begin() + min_bytes, r->End());
return true;
}
}
DCHECK_ALIGNED(tlab_start, kRegionSize);
Region* r = RefToRegionLocked(reinterpret_cast<mirror::Object*>(tlab_start));
DCHECK(r->IsAllocated());
- DCHECK_EQ(thread->GetThreadLocalBytesAllocated(), kRegionSize);
+ DCHECK_LE(thread->GetThreadLocalBytesAllocated(), kRegionSize);
r->RecordThreadLocalAllocations(thread->GetThreadLocalObjectsAllocated(),
thread->GetThreadLocalBytesAllocated());
r->is_a_tlab_ = false;
r->thread_ = nullptr;
}
- thread->SetTlab(nullptr, nullptr);
+ thread->SetTlab(nullptr, nullptr, nullptr);
}
size_t RegionSpace::RevokeAllThreadLocalBuffers() {
}
void RecordAlloc(mirror::Object* ref) REQUIRES(!region_lock_);
- bool AllocNewTlab(Thread* self) REQUIRES(!region_lock_);
+ bool AllocNewTlab(Thread* self, size_t min_bytes) REQUIRES(!region_lock_);
uint32_t Time() {
return time_;
return live_bytes_;
}
- size_t BytesAllocated() const {
- if (IsLarge()) {
- DCHECK_LT(begin_ + kRegionSize, Top());
- return static_cast<size_t>(Top() - begin_);
- } else if (IsLargeTail()) {
- DCHECK_EQ(begin_, Top());
- return 0;
- } else {
- DCHECK(IsAllocated()) << static_cast<uint>(state_);
- DCHECK_LE(begin_, Top());
- size_t bytes = static_cast<size_t>(Top() - begin_);
- DCHECK_LE(bytes, kRegionSize);
- return bytes;
- }
- }
+ size_t BytesAllocated() const;
size_t ObjectsAllocated() const {
if (IsLarge()) {
DCHECK_EQ(Top(), end_);
objects_allocated_.StoreRelaxed(num_objects);
top_.StoreRelaxed(begin_ + num_bytes);
- DCHECK_EQ(Top(), end_);
+ DCHECK_LE(Top(), end_);
}
private:
class PACKED(4) OatHeader {
public:
static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' };
- static constexpr uint8_t kOatVersion[] = { '1', '1', '8', '\0' }; // ARM64 Read barriers thunks.
+ static constexpr uint8_t kOatVersion[] = { '1', '1', '9', '\0' }; // Add thread_local_limit.
static constexpr const char* kImageLocationKey = "image-location";
static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline";
}
}
-inline size_t Thread::TlabSize() const {
- return tlsPtr_.thread_local_end - tlsPtr_.thread_local_pos;
-}
-
inline mirror::Object* Thread::AllocTlab(size_t bytes) {
DCHECK_GE(TlabSize(), bytes);
++tlsPtr_.thread_local_objects;
}
}
-void Thread::SetTlab(uint8_t* start, uint8_t* end) {
+void Thread::SetTlab(uint8_t* start, uint8_t* end, uint8_t* limit) {
DCHECK_LE(start, end);
+ DCHECK_LE(end, limit);
tlsPtr_.thread_local_start = start;
tlsPtr_.thread_local_pos = tlsPtr_.thread_local_start;
tlsPtr_.thread_local_end = end;
+ tlsPtr_.thread_local_limit = limit;
tlsPtr_.thread_local_objects = 0;
}
void ResetQuickAllocEntryPointsForThread(bool is_marking);
// Returns the remaining space in the TLAB.
- size_t TlabSize() const;
+ size_t TlabSize() const {
+ return tlsPtr_.thread_local_end - tlsPtr_.thread_local_pos;
+ }
+
+ // Returns the remaining space in the TLAB if we were to expand it to maximum capacity.
+ size_t TlabRemainingCapacity() const {
+ return tlsPtr_.thread_local_limit - tlsPtr_.thread_local_pos;
+ }
+
+ // Expand the TLAB by a fixed number of bytes. There must be enough capacity to do so.
+ void ExpandTlab(size_t bytes) {
+ tlsPtr_.thread_local_end += bytes;
+ DCHECK_LE(tlsPtr_.thread_local_end, tlsPtr_.thread_local_limit);
+ }
+
// Doesn't check that there is room.
mirror::Object* AllocTlab(size_t bytes);
- void SetTlab(uint8_t* start, uint8_t* end);
+ void SetTlab(uint8_t* start, uint8_t* end, uint8_t* limit);
bool HasTlab() const;
uint8_t* GetTlabStart() {
return tlsPtr_.thread_local_start;
frame_id_to_shadow_frame(nullptr), name(nullptr), pthread_self(0),
last_no_thread_suspension_cause(nullptr), checkpoint_function(nullptr),
thread_local_start(nullptr), thread_local_pos(nullptr), thread_local_end(nullptr),
+ thread_local_limit(nullptr),
thread_local_objects(0), mterp_current_ibase(nullptr), mterp_default_ibase(nullptr),
mterp_alt_ibase(nullptr), thread_local_alloc_stack_top(nullptr),
thread_local_alloc_stack_end(nullptr),
uint8_t* thread_local_pos;
uint8_t* thread_local_end;
+ // Thread local limit is how much we can expand the thread local buffer to, it is greater or
+ // equal to thread_local_end.
+ uint8_t* thread_local_limit;
+
size_t thread_local_objects;
// Entrypoint function pointers.
OSDN Git Service
