1 // MT-optimized allocator -*- C++ -*-
3 // Copyright (C) 2003, 2004 Free Software Foundation, Inc.
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 2, or (at your option)
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
16 // You should have received a copy of the GNU General Public License along
17 // with this library; see the file COPYING. If not, write to the Free
18 // Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307,
21 // As a special exception, you may use this file as part of a free software
22 // library without restriction. Specifically, if other files instantiate
23 // templates or use macros or inline functions from this file, or you compile
24 // this file and link it with other files to produce an executable, this
25 // file does not by itself cause the resulting executable to be covered by
26 // the GNU General Public License. This exception does not however
27 // invalidate any other reasons why the executable file might be covered by
28 // the GNU General Public License.
30 /** @file ext/mt_allocator.h
31 * This file is a GNU extension to the Standard C++ Library.
32 * You should only include this header if you are using GCC 3 or later.
35 #ifndef _MT_ALLOCATOR_H
36 #define _MT_ALLOCATOR_H 1
40 #include <bits/functexcept.h>
41 #include <bits/gthr.h>
42 #include <bits/atomicity.h>
47 * This is a fixed size (power of 2) allocator which - when
48 * compiled with thread support - will maintain one freelist per
49 * size per thread plus a "global" one. Steps are taken to limit
50 * the per thread freelist sizes (by returning excess back to
54 * http://gcc.gnu.org/onlinedocs/libstdc++/ext/mt_allocator.html
56 template<typename _Tp>
60 typedef size_t size_type;
61 typedef ptrdiff_t difference_type;
63 typedef const _Tp* const_pointer;
64 typedef _Tp& reference;
65 typedef const _Tp& const_reference;
66 typedef _Tp value_type;
68 template<typename _Tp1>
70 { typedef __mt_alloc<_Tp1> other; };
77 __mt_alloc(const __mt_alloc&) throw()
82 template<typename _Tp1>
83 __mt_alloc(const __mt_alloc<_Tp1>& obj) throw()
88 ~__mt_alloc() throw() { }
91 address(reference __x) const
95 address(const_reference __x) const
99 max_size() const throw()
100 { return size_t(-1) / sizeof(_Tp); }
102 // _GLIBCXX_RESOLVE_LIB_DEFECTS
103 // 402. wrong new expression in [some_] allocator::construct
105 construct(pointer __p, const _Tp& __val)
106 { ::new(__p) _Tp(__val); }
109 destroy(pointer __p) { __p->~_Tp(); }
112 allocate(size_type __n, const void* = 0);
115 deallocate(pointer __p, size_type __n);
117 // Variables used to configure the behavior of the allocator,
118 // assigned and explained in detail below.
122 // NB: In any case must be >= sizeof(_Block_record), that
123 // is 4 on 32 bit machines and 8 on 64 bit machines.
126 // Allocation requests (after round-up to power of 2) below
127 // this value will be handled by the allocator. A raw new/
128 // call will be used for requests larger than this value.
131 // Size in bytes of the smallest bin.
132 // NB: Must be a power of 2 and >= _M_align.
135 // In order to avoid fragmenting and minimize the number of
136 // new() calls we always request new memory using this
137 // value. Based on previous discussions on the libstdc++
138 // mailing list we have choosen the value below.
139 // See http://gcc.gnu.org/ml/libstdc++/2001-07/msg00077.html
140 size_t _M_chunk_size;
142 // The maximum number of supported threads. For
143 // single-threaded operation, use one. Maximum values will
144 // vary depending on details of the underlying system. (For
145 // instance, Linux 2.4.18 reports 4070 in
146 // /proc/sys/kernel/threads-max, while Linux 2.6.6 reports
148 size_t _M_max_threads;
150 // Each time a deallocation occurs in a threaded application
151 // we make sure that there are no more than
152 // _M_freelist_headroom % of used memory on the freelist. If
153 // the number of additional records is more than
154 // _M_freelist_headroom % of the freelist, we move these
155 // records back to the global pool.
156 size_t _M_freelist_headroom;
158 // Set to true forces all allocations to use new().
163 : _M_align(8), _M_max_bytes(128), _M_min_bin(8),
164 _M_chunk_size(4096 - 4 * sizeof(void*)),
165 _M_max_threads(4096), _M_freelist_headroom(10),
166 _M_force_new(getenv("GLIBCXX_FORCE_NEW") ? true : false)
170 _Tune(size_t __align, size_t __maxb, size_t __minbin,
171 size_t __chunk, size_t __maxthreads, size_t __headroom,
173 : _M_align(__align), _M_max_bytes(__maxb), _M_min_bin(__minbin),
174 _M_chunk_size(__chunk), _M_max_threads(__maxthreads),
175 _M_freelist_headroom(__headroom), _M_force_new(__force)
181 { return _S_options; }
184 _S_set_options(_Tune __t)
191 // We need to create the initial lists and set up some variables
192 // before we can answer to the first request for memory.
194 static __gthread_once_t _S_once;
201 // Configuration options.
202 static _Tune _S_options;
204 // Using short int as type for the binmap implies we are never
205 // caching blocks larger than 65535 with this allocator
206 typedef unsigned short int _Binmap_type;
207 static _Binmap_type* _S_binmap;
209 // Each requesting thread is assigned an id ranging from 1 to
210 // _S_max_threads. Thread id 0 is used as a global memory pool.
211 // In order to get constant performance on the thread assignment
212 // routine, we keep a list of free ids. When a thread first
213 // requests memory we remove the first record in this list and
214 // stores the address in a __gthread_key. When initializing the
215 // __gthread_key we specify a destructor. When this destructor
216 // (i.e. the thread dies) is called, we return the thread id to
217 // the front of this list.
219 struct _Thread_record
221 // Points to next free thread id record. NULL if last record in list.
222 _Thread_record* volatile _M_next;
224 // Thread id ranging from 1 to _S_max_threads.
228 static _Thread_record* volatile _S_thread_freelist_first;
229 static __gthread_mutex_t _S_thread_freelist_mutex;
230 static __gthread_key_t _S_thread_key;
233 _S_destroy_thread_key(void* __freelist_pos);
241 // Points to the block_record of the next free block.
242 _Block_record* volatile _M_next;
245 // The thread id of the thread which has requested this block.
252 // An "array" of pointers to the first free block for each
253 // thread id. Memory to this "array" is allocated in _S_initialize()
254 // for _S_max_threads + global pool 0.
255 _Block_record** volatile _M_first;
258 // An "array" of counters used to keep track of the amount of
259 // blocks that are on the freelist/used for each thread id.
260 // Memory to these "arrays" is allocated in _S_initialize() for
261 // _S_max_threads + global pool 0.
262 size_t* volatile _M_free;
263 size_t* volatile _M_used;
265 // Each bin has its own mutex which is used to ensure data
266 // integrity while changing "ownership" on a block. The mutex
267 // is initialized in _S_initialize().
268 __gthread_mutex_t* _M_mutex;
272 // An "array" of bin_records each of which represents a specific
273 // power of 2 size. Memory to this "array" is allocated in
275 static _Bin_record* volatile _S_bin;
277 // Actual value calculated in _S_initialize().
278 static size_t _S_bin_size;
281 template<typename _Tp>
282 typename __mt_alloc<_Tp>::pointer
284 allocate(size_type __n, const void*)
286 // Although the test in __gthread_once() would suffice, we wrap
287 // test of the once condition in our own unlocked check. This
288 // saves one function call to pthread_once() (which itself only
289 // tests for the once value unlocked anyway and immediately
294 if (__gthread_active_p())
295 __gthread_once(&_S_once, _S_initialize);
301 // Requests larger than _M_max_bytes are handled by new/delete
303 const size_t __bytes = __n * sizeof(_Tp);
304 if (__bytes > _S_options._M_max_bytes || _S_options._M_force_new)
306 void* __ret = ::operator new(__bytes);
307 return static_cast<_Tp*>(__ret);
310 // Round up to power of 2 and figure out which bin to use.
311 const size_t __which = _S_binmap[__bytes];
312 const size_t __thread_id = _S_get_thread_id();
314 // Find out if we have blocks on our freelist. If so, go ahead
315 // and use them directly without having to lock anything.
316 const _Bin_record& __bin = _S_bin[__which];
317 _Block_record* __block = NULL;
318 if (__bin._M_first[__thread_id] == NULL)
320 // NB: For alignment reasons, we can't use the first _M_align
321 // bytes, even when sizeof(_Block_record) < _M_align.
322 const size_t __bin_size = ((_S_options._M_min_bin << __which)
323 + _S_options._M_align);
324 size_t __block_count = _S_options._M_chunk_size / __bin_size;
326 // Are we using threads?
327 // - Yes, check if there are free blocks on the global
328 // list. If so, grab up to __block_count blocks in one
329 // lock and change ownership. If the global list is
330 // empty, we allocate a new chunk and add those blocks
331 // directly to our own freelist (with us as owner).
332 // - No, all operations are made directly to global pool 0
333 // no need to lock or change ownership but check for free
334 // blocks on global list (and if not add new ones) and
335 // get the first one.
337 if (__gthread_active_p())
339 __gthread_mutex_lock(__bin._M_mutex);
340 if (__bin._M_first[0] == NULL)
342 // No need to hold the lock when we are adding a
343 // whole chunk to our own list.
344 __gthread_mutex_unlock(__bin._M_mutex);
346 void* __v = ::operator new(_S_options._M_chunk_size);
347 __bin._M_first[__thread_id] = static_cast<_Block_record*>(__v);
348 __bin._M_free[__thread_id] = __block_count;
351 __block = __bin._M_first[__thread_id];
352 while (__block_count-- > 0)
354 char* __c = reinterpret_cast<char*>(__block) + __bin_size;
355 __block->_M_next = reinterpret_cast<_Block_record*>(__c);
356 __block = __block->_M_next;
358 __block->_M_next = NULL;
362 // Is the number of required blocks greater than or
363 // equal to the number that can be provided by the
365 __bin._M_first[__thread_id] = __bin._M_first[0];
366 if (__block_count >= __bin._M_free[0])
368 __bin._M_free[__thread_id] = __bin._M_free[0];
369 __bin._M_free[0] = 0;
370 __bin._M_first[0] = NULL;
374 __bin._M_free[__thread_id] = __block_count;
375 __bin._M_free[0] -= __block_count;
377 __block = __bin._M_first[0];
378 while (__block_count-- > 0)
379 __block = __block->_M_next;
380 __bin._M_first[0] = __block->_M_next;
381 __block->_M_next = NULL;
383 __gthread_mutex_unlock(__bin._M_mutex);
389 void* __v = ::operator new(_S_options._M_chunk_size);
390 __bin._M_first[0] = static_cast<_Block_record*>(__v);
393 __block = __bin._M_first[0];
394 while (__block_count-- > 0)
396 char* __c = reinterpret_cast<char*>(__block) + __bin_size;
397 __block->_M_next = reinterpret_cast<_Block_record*>(__c);
398 __block = __block->_M_next;
400 __block->_M_next = NULL;
404 __block = __bin._M_first[__thread_id];
405 __bin._M_first[__thread_id] = __bin._M_first[__thread_id]->_M_next;
407 if (__gthread_active_p())
409 __block->_M_thread_id = __thread_id;
410 --__bin._M_free[__thread_id];
411 ++__bin._M_used[__thread_id];
415 char* __c = reinterpret_cast<char*>(__block) + _S_options._M_align;
416 return static_cast<_Tp*>(static_cast<void*>(__c));
419 template<typename _Tp>
422 deallocate(pointer __p, size_type __n)
424 // Requests larger than _M_max_bytes are handled by operators
425 // new/delete directly.
426 const size_t __bytes = __n * sizeof(_Tp);
427 if (__bytes > _S_options._M_max_bytes || _S_options._M_force_new)
429 ::operator delete(__p);
433 // Round up to power of 2 and figure out which bin to use.
434 const size_t __which = _S_binmap[__bytes];
435 const _Bin_record& __bin = _S_bin[__which];
437 char* __c = reinterpret_cast<char*>(__p) - _S_options._M_align;
438 _Block_record* __block = reinterpret_cast<_Block_record*>(__c);
441 if (__gthread_active_p())
443 // Calculate the number of records to remove from our freelist:
444 // in order to avoid too much contention we wait until the
445 // number of records is "high enough".
446 const size_t __thread_id = _S_get_thread_id();
448 long __remove = ((__bin._M_free[__thread_id]
449 * _S_options._M_freelist_headroom)
450 - __bin._M_used[__thread_id]);
451 if (__remove > static_cast<long>(100 * (_S_bin_size - __which)
452 * _S_options._M_freelist_headroom)
453 && __remove > static_cast<long>(__bin._M_free[__thread_id]))
455 _Block_record* __tmp = __bin._M_first[__thread_id];
456 _Block_record* __first = __tmp;
457 __remove /= _S_options._M_freelist_headroom;
458 const long __removed = __remove;
460 while (__remove-- > 0)
461 __tmp = __tmp->_M_next;
462 __bin._M_first[__thread_id] = __tmp->_M_next;
463 __bin._M_free[__thread_id] -= __removed;
465 __gthread_mutex_lock(__bin._M_mutex);
466 __tmp->_M_next = __bin._M_first[0];
467 __bin._M_first[0] = __first;
468 __bin._M_free[0] += __removed;
469 __gthread_mutex_unlock(__bin._M_mutex);
472 // Return this block to our list and update counters and
473 // owner id as needed.
474 --__bin._M_used[__block->_M_thread_id];
476 __block->_M_next = __bin._M_first[__thread_id];
477 __bin._M_first[__thread_id] = __block;
479 ++__bin._M_free[__thread_id];
484 // Single threaded application - return to global pool.
485 __block->_M_next = __bin._M_first[0];
486 __bin._M_first[0] = __block;
490 template<typename _Tp>
495 // This method is called on the first allocation (when _S_init is still
496 // false) to create the bins.
498 // Ensure that the static initialization of _S_options has
499 // happened. This depends on (a) _M_align == 0 being an invalid
500 // value that is only present at startup, and (b) the real
501 // static initialization that happens later not actually
502 // changing anything.
503 if (_S_options._M_align == 0)
504 new (&_S_options) _Tune;
506 // _M_force_new must not change after the first allocate(),
507 // which in turn calls this method, so if it's false, it's false
508 // forever and we don't need to return here ever again.
509 if (_S_options._M_force_new)
515 // Calculate the number of bins required based on _M_max_bytes.
516 // _S_bin_size is statically-initialized to one.
517 size_t __bin_size = _S_options._M_min_bin;
518 while (_S_options._M_max_bytes > __bin_size)
524 // Setup the bin map for quick lookup of the relevant bin.
525 const size_t __j = (_S_options._M_max_bytes + 1) * sizeof(_Binmap_type);
526 _S_binmap = static_cast<_Binmap_type*>(::operator new(__j));
528 _Binmap_type* __bp = _S_binmap;
529 _Binmap_type __bin_max = _S_options._M_min_bin;
530 _Binmap_type __bint = 0;
531 for (_Binmap_type __ct = 0; __ct <= _S_options._M_max_bytes; ++__ct)
533 if (__ct > __bin_max)
541 // Initialize _S_bin and its members.
542 void* __v = ::operator new(sizeof(_Bin_record) * _S_bin_size);
543 _S_bin = static_cast<_Bin_record*>(__v);
545 // If __gthread_active_p() create and initialize the list of
546 // free thread ids. Single threaded applications use thread id 0
547 // directly and have no need for this.
549 if (__gthread_active_p())
551 const size_t __k = sizeof(_Thread_record) * _S_options._M_max_threads;
552 __v = ::operator new(__k);
553 _S_thread_freelist_first = static_cast<_Thread_record*>(__v);
555 // NOTE! The first assignable thread id is 1 since the
556 // global pool uses id 0
558 for (__i = 1; __i < _S_options._M_max_threads; ++__i)
560 _Thread_record& __tr = _S_thread_freelist_first[__i - 1];
561 __tr._M_next = &_S_thread_freelist_first[__i];
566 _S_thread_freelist_first[__i - 1]._M_next = NULL;
567 _S_thread_freelist_first[__i - 1]._M_id = __i;
569 // Make sure this is initialized.
570 #ifndef __GTHREAD_MUTEX_INIT
571 __GTHREAD_MUTEX_INIT_FUNCTION(&_S_thread_freelist_mutex);
573 // Initialize per thread key to hold pointer to
574 // _S_thread_freelist.
575 __gthread_key_create(&_S_thread_key, _S_destroy_thread_key);
577 const size_t __max_threads = _S_options._M_max_threads + 1;
578 for (size_t __n = 0; __n < _S_bin_size; ++__n)
580 _Bin_record& __bin = _S_bin[__n];
581 __v = ::operator new(sizeof(_Block_record*) * __max_threads);
582 __bin._M_first = static_cast<_Block_record**>(__v);
584 __v = ::operator new(sizeof(size_t) * __max_threads);
585 __bin._M_free = static_cast<size_t*>(__v);
587 __v = ::operator new(sizeof(size_t) * __max_threads);
588 __bin._M_used = static_cast<size_t*>(__v);
590 __v = ::operator new(sizeof(__gthread_mutex_t));
591 __bin._M_mutex = static_cast<__gthread_mutex_t*>(__v);
593 #ifdef __GTHREAD_MUTEX_INIT
595 // Do not copy a POSIX/gthr mutex once in use.
596 __gthread_mutex_t __tmp = __GTHREAD_MUTEX_INIT;
597 *__bin._M_mutex = __tmp;
600 { __GTHREAD_MUTEX_INIT_FUNCTION(__bin._M_mutex); }
603 for (size_t __threadn = 0; __threadn < __max_threads;
606 __bin._M_first[__threadn] = NULL;
607 __bin._M_free[__threadn] = 0;
608 __bin._M_used[__threadn] = 0;
614 for (size_t __n = 0; __n < _S_bin_size; ++__n)
616 _Bin_record& __bin = _S_bin[__n];
617 __v = ::operator new(sizeof(_Block_record*));
618 __bin._M_first = static_cast<_Block_record**>(__v);
619 __bin._M_first[0] = NULL;
625 template<typename _Tp>
631 // If we have thread support and it's active we check the thread
632 // key value and return its id or if it's not set we take the
633 // first record from _S_thread_freelist and sets the key and
635 if (__gthread_active_p())
637 _Thread_record* __freelist_pos =
638 static_cast<_Thread_record*>(__gthread_getspecific(_S_thread_key));
639 if (__freelist_pos == NULL)
641 // Since _S_options._M_max_threads must be larger than
642 // the theoretical max number of threads of the OS the
643 // list can never be empty.
644 __gthread_mutex_lock(&_S_thread_freelist_mutex);
645 __freelist_pos = _S_thread_freelist_first;
646 _S_thread_freelist_first = _S_thread_freelist_first->_M_next;
647 __gthread_mutex_unlock(&_S_thread_freelist_mutex);
649 __gthread_setspecific(_S_thread_key,
650 static_cast<void*>(__freelist_pos));
652 return __freelist_pos->_M_id;
655 // Otherwise (no thread support or inactive) all requests are
656 // served from the global pool 0.
661 template<typename _Tp>
664 _S_destroy_thread_key(void* __freelist_pos)
666 // Return this thread id record to front of thread_freelist.
667 __gthread_mutex_lock(&_S_thread_freelist_mutex);
668 _Thread_record* __tr = static_cast<_Thread_record*>(__freelist_pos);
669 __tr->_M_next = _S_thread_freelist_first;
670 _S_thread_freelist_first = __tr;
671 __gthread_mutex_unlock(&_S_thread_freelist_mutex);
675 template<typename _Tp>
677 operator==(const __mt_alloc<_Tp>&, const __mt_alloc<_Tp>&)
680 template<typename _Tp>
682 operator!=(const __mt_alloc<_Tp>&, const __mt_alloc<_Tp>&)
685 template<typename _Tp>
686 bool __mt_alloc<_Tp>::_S_init = false;
688 template<typename _Tp>
689 typename __mt_alloc<_Tp>::_Tune __mt_alloc<_Tp>::_S_options;
691 template<typename _Tp>
692 typename __mt_alloc<_Tp>::_Binmap_type* __mt_alloc<_Tp>::_S_binmap;
694 template<typename _Tp>
695 typename __mt_alloc<_Tp>::_Bin_record* volatile __mt_alloc<_Tp>::_S_bin;
697 template<typename _Tp>
698 size_t __mt_alloc<_Tp>::_S_bin_size = 1;
700 // Actual initialization in _S_initialize().
702 template<typename _Tp>
703 __gthread_once_t __mt_alloc<_Tp>::_S_once = __GTHREAD_ONCE_INIT;
705 template<typename _Tp>
706 typename __mt_alloc<_Tp>::_Thread_record*
707 volatile __mt_alloc<_Tp>::_S_thread_freelist_first = NULL;
709 template<typename _Tp>
710 __gthread_key_t __mt_alloc<_Tp>::_S_thread_key;
712 template<typename _Tp>
714 #ifdef __GTHREAD_MUTEX_INIT
715 __mt_alloc<_Tp>::_S_thread_freelist_mutex = __GTHREAD_MUTEX_INIT;
717 __mt_alloc<_Tp>::_S_thread_freelist_mutex;
720 } // namespace __gnu_cxx