From 98b816c625bec979283b84663cba652b79ce298e Mon Sep 17 00:00:00 2001 From: YuQing <384681@qq.com> Date: Tue, 1 Sep 2020 21:22:13 +0800 Subject: [PATCH] fast_mblock.[hc]: support alloc elements limit --- HISTORY | 3 +- src/common_blocked_queue.c | 9 +++-- src/connection_pool.c | 8 ++-- src/fast_allocator.c | 5 ++- src/fast_mblock.c | 78 ++++++++++++++++++++++++++++---------- src/fast_mblock.h | 21 +++++++--- src/flat_skiplist.c | 8 +++- src/multi_skiplist.c | 12 ++++-- src/shared_buffer.c | 7 ++-- src/skiplist_set.c | 8 +++- src/uniq_skiplist.c | 8 ++-- 11 files changed, 118 insertions(+), 49 deletions(-) diff --git a/HISTORY b/HISTORY index 37ac67b..f4779ce 100644 --- a/HISTORY +++ b/HISTORY @@ -1,5 +1,5 @@ -Version 1.44 2020-08-31 +Version 1.44 2020-09-01 * add test file src/tests/test_pthread_lock.c * add uniq_skiplist.[hc] * add function split_string_ex @@ -35,6 +35,7 @@ Version 1.44 2020-08-31 * add files: shared_buffer.[hc] * add files: thread_pool.[hc] * shared_func.[hc]: add function fc_path_contains + * fast_mblock.[hc]: support alloc elements limit Version 1.43 2019-12-25 * replace function call system to getExecResult, diff --git a/src/common_blocked_queue.c b/src/common_blocked_queue.c index 92ad2c4..e36725f 100644 --- a/src/common_blocked_queue.c +++ b/src/common_blocked_queue.c @@ -11,6 +11,7 @@ int common_blocked_queue_init_ex(struct common_blocked_queue *queue, const int alloc_elements_once) { + const int64_t alloc_elements_limit = 0; int result; if ((result=init_pthread_lock(&queue->lock)) != 0) @@ -30,10 +31,10 @@ int common_blocked_queue_init_ex(struct common_blocked_queue *queue, return result; } - if ((result=fast_mblock_init_ex2(&queue->mblock, - "queue_node", sizeof(struct common_blocked_node), - alloc_elements_once, NULL, NULL, false, - NULL, NULL, NULL)) != 0) + if ((result=fast_mblock_init_ex1(&queue->mblock, "queue_node", + sizeof(struct common_blocked_node), + alloc_elements_once, alloc_elements_limit, + NULL, NULL, false)) != 0) { return result; } diff --git a/src/connection_pool.c b/src/connection_pool.c index 0dfefe5..a062e60 100644 --- a/src/connection_pool.c +++ b/src/connection_pool.c @@ -22,6 +22,7 @@ int conn_pool_init_ex1(ConnectionPool *cp, int connect_timeout, fc_connection_callback_func validate_func, void *validate_args, const int extra_data_size) { + const int64_t alloc_elements_limit = 0; int result; int init_capacity; @@ -40,15 +41,16 @@ int conn_pool_init_ex1(ConnectionPool *cp, int connect_timeout, init_capacity = htable_init_capacity > 0 ? htable_init_capacity : 256; if ((result=fast_mblock_init_ex1(&cp->manager_allocator, "cpool_manager", - sizeof(ConnectionManager), init_capacity, NULL, NULL, - false)) != 0) + sizeof(ConnectionManager), init_capacity, + alloc_elements_limit, NULL, NULL, false)) != 0) { return result; } if ((result=fast_mblock_init_ex1(&cp->node_allocator, "cpool_node", sizeof(ConnectionNode) + sizeof(ConnectionInfo) + - extra_data_size, init_capacity, NULL, NULL, false)) != 0) + extra_data_size, init_capacity, alloc_elements_limit, + NULL, NULL, false)) != 0) { return result; } diff --git a/src/fast_allocator.c b/src/fast_allocator.c index f818735..3a1229b 100644 --- a/src/fast_allocator.c +++ b/src/fast_allocator.c @@ -128,6 +128,7 @@ static int allocator_array_check_capacity(struct fast_allocator_context *acontex static int region_init(struct fast_allocator_context *acontext, const char *mblock_name_prefix, struct fast_region_info *region) { + const int64_t alloc_elements_limit = 0; int result; int bytes; int element_size; @@ -167,8 +168,8 @@ static int region_init(struct fast_allocator_context *acontext, name = NULL; } result = fast_mblock_init_ex2(&allocator->mblock, name, element_size, - region->alloc_elements_once, NULL, NULL, acontext->need_lock, - fast_allocator_malloc_trunk_check, + region->alloc_elements_once, alloc_elements_limit, NULL, NULL, + acontext->need_lock, fast_allocator_malloc_trunk_check, fast_allocator_malloc_trunk_notify_func, acontext); if (result != 0) { diff --git a/src/fast_mblock.c b/src/fast_mblock.c index 9c3067e..081256f 100644 --- a/src/fast_mblock.c +++ b/src/fast_mblock.c @@ -23,6 +23,9 @@ struct _fast_mblock_manager static struct _fast_mblock_manager mblock_manager = {false, 0}; +#define fast_mblock_get_trunk_size(mblock, block_size, element_count) \ + (sizeof(struct fast_mblock_malloc) + block_size * element_count) + int fast_mblock_manager_init() { int result; @@ -337,16 +340,18 @@ int fast_mblock_manager_stat_print_ex(const bool hide_empty, const int order_by) int fast_mblock_init_ex(struct fast_mblock_man *mblock, const int element_size, const int alloc_elements_once, + const int64_t alloc_elements_limit, fast_mblock_alloc_init_func init_func, void *init_args, const bool need_lock) { return fast_mblock_init_ex2(mblock, NULL, element_size, - alloc_elements_once, init_func, init_args, - need_lock, NULL, NULL, NULL); + alloc_elements_once, alloc_elements_limit, init_func, + init_args, need_lock, NULL, NULL, NULL); } int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name, const int element_size, const int alloc_elements_once, + const int64_t alloc_elements_limit, fast_mblock_alloc_init_func init_func, void *init_args, const bool need_lock, fast_mblock_malloc_trunk_check_func malloc_trunk_check, @@ -365,15 +370,21 @@ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name, } mblock->info.element_size = MEM_ALIGN(element_size); + mblock->alloc_elements.limit = alloc_elements_limit; block_size = fast_mblock_get_block_size(mblock); if (alloc_elements_once > 0) { - mblock->alloc_elements_once = alloc_elements_once; + mblock->alloc_elements.once = alloc_elements_once; } else { - mblock->alloc_elements_once = (1024 * 1024) / block_size; + mblock->alloc_elements.once = (1024 * 1024) / block_size; } + if (mblock->alloc_elements.limit > 0 && mblock->alloc_elements.once > + mblock->alloc_elements.limit) + { + mblock->alloc_elements.once = mblock->alloc_elements.limit; + } if (need_lock && (result=init_pthread_lock(&(mblock->lock))) != 0) { @@ -395,8 +406,8 @@ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name, mblock->info.element_total_count = 0; mblock->info.element_used_count = 0; mblock->info.instance_count = 1; - mblock->info.trunk_size = sizeof(struct fast_mblock_malloc) + block_size * - mblock->alloc_elements_once; + mblock->info.trunk_size = fast_mblock_get_trunk_size(mblock, + block_size, mblock->alloc_elements.once); mblock->need_lock = need_lock; mblock->malloc_trunk_callback.check_func = malloc_trunk_check; mblock->malloc_trunk_callback.notify_func = malloc_trunk_notify; @@ -425,27 +436,52 @@ static int fast_mblock_prealloc(struct fast_mblock_man *mblock) char *pLast; int result; int block_size; + int trunk_size; + int alloc_count; block_size = fast_mblock_get_block_size(mblock); + if (mblock->alloc_elements.limit > 0) + { + int64_t avail_count; + avail_count = mblock->alloc_elements.limit - + mblock->info.element_total_count; + if (avail_count <= 0) + { + logError("file: "__FILE__", line: %d, " + "allocated elements exceed limit: %"PRId64, + __LINE__, mblock->alloc_elements.limit); + return EOVERFLOW; + } + + alloc_count = avail_count > mblock->alloc_elements.once ? + mblock->alloc_elements.once : avail_count; + trunk_size = fast_mblock_get_trunk_size(mblock, + block_size, alloc_count); + } + else + { + alloc_count = mblock->alloc_elements.once; + trunk_size = mblock->info.trunk_size; + } + if (mblock->malloc_trunk_callback.check_func != NULL && - mblock->malloc_trunk_callback.check_func( - mblock->info.trunk_size, - mblock->malloc_trunk_callback.args) != 0) + mblock->malloc_trunk_callback.check_func(trunk_size, + mblock->malloc_trunk_callback.args) != 0) { return ENOMEM; } - pNew = (char *)fc_malloc(mblock->info.trunk_size); + pNew = (char *)fc_malloc(trunk_size); if (pNew == NULL) { return ENOMEM; } - memset(pNew, 0, mblock->info.trunk_size); + memset(pNew, 0, trunk_size); pMallocNode = (struct fast_mblock_malloc *)pNew; pTrunkStart = pNew + sizeof(struct fast_mblock_malloc); - pLast = pNew + (mblock->info.trunk_size - block_size); + pLast = pNew + (trunk_size - block_size); for (p=pTrunkStart; p<=pLast; p += block_size) { pNode = (struct fast_mblock_node *)p; @@ -466,18 +502,20 @@ static int fast_mblock_prealloc(struct fast_mblock_man *mblock) mblock->free_chain_head = (struct fast_mblock_node *)pTrunkStart; pMallocNode->ref_count = 0; + pMallocNode->alloc_count = alloc_count; + pMallocNode->trunk_size = trunk_size; pMallocNode->prev = mblock->trunks.head.prev; pMallocNode->next = &mblock->trunks.head; mblock->trunks.head.prev->next = pMallocNode; mblock->trunks.head.prev = pMallocNode; mblock->info.trunk_total_count++; - mblock->info.element_total_count += mblock->alloc_elements_once; + mblock->info.element_total_count += alloc_count; if (mblock->malloc_trunk_callback.notify_func != NULL) { - mblock->malloc_trunk_callback.notify_func(mblock->info.trunk_size, - mblock->malloc_trunk_callback.args); + mblock->malloc_trunk_callback.notify_func(trunk_size, + mblock->malloc_trunk_callback.args); } return 0; @@ -489,11 +527,11 @@ static inline void fast_mblock_remove_trunk(struct fast_mblock_man *mblock, pMallocNode->prev->next = pMallocNode->next; pMallocNode->next->prev = pMallocNode->prev; mblock->info.trunk_total_count--; - mblock->info.element_total_count -= mblock->alloc_elements_once; + mblock->info.element_total_count -= pMallocNode->alloc_count; if (mblock->malloc_trunk_callback.notify_func != NULL) { - mblock->malloc_trunk_callback.notify_func(-1 * mblock->info.trunk_size, + mblock->malloc_trunk_callback.notify_func(-1 * pMallocNode->trunk_size, mblock->malloc_trunk_callback.args); } } @@ -570,9 +608,9 @@ struct fast_mblock_node *fast_mblock_alloc(struct fast_mblock_man *mblock) if (mblock->need_lock && (result=pthread_mutex_lock(&(mblock->lock))) != 0) { - logError("file: "__FILE__", line: %d, " \ - "call pthread_mutex_lock fail, " \ - "errno: %d, error info: %s", \ + logError("file: "__FILE__", line: %d, " + "call pthread_mutex_lock fail, " + "errno: %d, error info: %s", __LINE__, result, STRERROR(result)); return NULL; } diff --git a/src/fast_mblock.h b/src/fast_mblock.h index 3b63534..d7acb0d 100644 --- a/src/fast_mblock.h +++ b/src/fast_mblock.h @@ -36,7 +36,9 @@ struct fast_mblock_node /* malloc chain */ struct fast_mblock_malloc { - int64_t ref_count; //refference count + int64_t ref_count; //refference count + int alloc_count; //allocated element count + int trunk_size; //trunk bytes struct fast_mblock_malloc *prev; struct fast_mblock_malloc *next; }; @@ -82,7 +84,10 @@ struct fast_mblock_malloc_trunk_callback struct fast_mblock_man { struct fast_mblock_info info; - int alloc_elements_once; //alloc elements once + struct { + int once; //alloc elements once + int64_t limit; //<= 0 for no limit + } alloc_elements; struct fast_mblock_node *free_chain_head; //free node chain struct fast_mblock_trunks trunks; struct fast_mblock_chain delay_free_chain; //delay free node chain @@ -112,7 +117,7 @@ extern "C" { #define fast_mblock_init(mblock, element_size, alloc_elements_once) \ fast_mblock_init_ex(mblock, element_size, alloc_elements_once, \ - NULL, NULL, true) + 0, NULL, NULL, true) /** mblock init @@ -120,6 +125,7 @@ parameters: mblock: the mblock pointer element_size: element size, such as sizeof(struct xxx) alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once + alloc_elements_limit: malloc elements limit, <= 0 for no limit init_func: the init function init_args: the args for init_func need_lock: if need lock @@ -127,6 +133,7 @@ return error no, 0 for success, != 0 fail */ int fast_mblock_init_ex(struct fast_mblock_man *mblock, const int element_size, const int alloc_elements_once, + const int64_t alloc_elements_limit, fast_mblock_alloc_init_func init_func, void *init_args, const bool need_lock); @@ -137,6 +144,7 @@ parameters: mblock: the mblock pointer element_size: element size, such as sizeof(struct xxx) alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once + alloc_elements_limit: malloc elements limit, <= 0 for no limit init_func: the init function init_args: the args for init_func need_lock: if need lock @@ -147,6 +155,7 @@ return error no, 0 for success, != 0 fail */ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name, const int element_size, const int alloc_elements_once, + const int64_t alloc_elements_limit, fast_mblock_alloc_init_func init_func, void *init_args, const bool need_lock, fast_mblock_malloc_trunk_check_func malloc_trunk_check, @@ -160,6 +169,7 @@ parameters: mblock: the mblock pointer element_size: element size, such as sizeof(struct xxx) alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once + alloc_elements_limit: malloc elements limit, <= 0 for no limit init_func: the init function init_args: the args for init_func need_lock: if need lock @@ -168,12 +178,13 @@ return error no, 0 for success, != 0 fail static inline int fast_mblock_init_ex1(struct fast_mblock_man *mblock, const char *name, const int element_size, const int alloc_elements_once, + const int64_t alloc_elements_limit, fast_mblock_alloc_init_func init_func, void *init_args, const bool need_lock) { return fast_mblock_init_ex2(mblock, name, element_size, - alloc_elements_once, init_func, init_args, - need_lock, NULL, NULL, NULL); + alloc_elements_once, alloc_elements_limit, init_func, + init_args, need_lock, NULL, NULL, NULL); } /** diff --git a/src/flat_skiplist.c b/src/flat_skiplist.c index 6ee479b..310ec74 100644 --- a/src/flat_skiplist.c +++ b/src/flat_skiplist.c @@ -22,6 +22,8 @@ int flat_skiplist_init_ex(FlatSkiplist *sl, const int level_count, skiplist_compare_func compare_func, skiplist_free_func free_func, const int min_alloc_elements_once) { + const int64_t alloc_elements_limit = 0; + char name[64]; int bytes; int element_size; int i; @@ -65,10 +67,12 @@ int flat_skiplist_init_ex(FlatSkiplist *sl, const int level_count, } for (i=level_count-1; i>=0; i--) { + sprintf(name, "flat-sl-level%02d", i); element_size = sizeof(FlatSkiplistNode) + sizeof(FlatSkiplistNode *) * (i + 1); - if ((result=fast_mblock_init_ex(sl->mblocks + i, - element_size, alloc_elements_once, NULL, NULL, false)) != 0) + if ((result=fast_mblock_init_ex1(sl->mblocks + i, name, + element_size, alloc_elements_once, alloc_elements_limit, + NULL, NULL, false)) != 0) { return result; } diff --git a/src/multi_skiplist.c b/src/multi_skiplist.c index 6749841..f1ed57c 100644 --- a/src/multi_skiplist.c +++ b/src/multi_skiplist.c @@ -23,6 +23,8 @@ int multi_skiplist_init_ex(MultiSkiplist *sl, const int level_count, skiplist_free_func free_func, const int min_alloc_elements_once) { + const int64_t alloc_elements_limit = 0; + char name[64]; int bytes; int element_size; int i; @@ -66,10 +68,12 @@ int multi_skiplist_init_ex(MultiSkiplist *sl, const int level_count, } for (i=level_count-1; i>=0; i--) { + sprintf(name, "multi-sl-level%02d", i); element_size = sizeof(MultiSkiplistNode) + sizeof(MultiSkiplistNode *) * (i + 1); - if ((result=fast_mblock_init_ex(sl->mblocks + i, - element_size, alloc_elements_once, NULL, NULL, false)) != 0) + if ((result=fast_mblock_init_ex1(sl->mblocks + i, name, + element_size, alloc_elements_once, alloc_elements_limit, + NULL, NULL, false)) != 0) { return result; } @@ -92,9 +96,9 @@ int multi_skiplist_init_ex(MultiSkiplist *sl, const int level_count, } memset(sl->tail, 0, sl->mblocks[0].info.element_size); - if ((result=fast_mblock_init_ex(&sl->data_mblock, + if ((result=fast_mblock_init_ex1(&sl->data_mblock, "multi-sl-data", sizeof(MultiSkiplistData), alloc_elements_once, - NULL, NULL, false)) != 0) + alloc_elements_limit, NULL, NULL, false)) != 0) { return result; } diff --git a/src/shared_buffer.c b/src/shared_buffer.c index 9f409c2..ccb7970 100644 --- a/src/shared_buffer.c +++ b/src/shared_buffer.c @@ -23,13 +23,14 @@ int shared_buffer_init_ex(SharedBufferContext *context, const int alloc_elements_once, const int buffer_init_capacity, const bool need_lock) { + const int64_t alloc_elements_limit = 0; int result; context->buffer_init_capacity = buffer_init_capacity; - if ((result=fast_mblock_init_ex2(&context->allocator, "shared_buffer", + if ((result=fast_mblock_init_ex1(&context->allocator, "shared_buffer", sizeof(SharedBuffer), alloc_elements_once, - shared_buffer_alloc_init, context, need_lock, - NULL, NULL, NULL)) != 0) + alloc_elements_limit, shared_buffer_alloc_init, + context, need_lock)) != 0) { return result; } diff --git a/src/skiplist_set.c b/src/skiplist_set.c index c678f97..dec7122 100644 --- a/src/skiplist_set.c +++ b/src/skiplist_set.c @@ -22,6 +22,8 @@ int skiplist_set_init_ex(SkiplistSet *sl, const int level_count, skiplist_compare_func compare_func, skiplist_free_func free_func, const int min_alloc_elements_once) { + const int64_t alloc_elements_limit = 0; + char name[64]; int bytes; int element_size; int i; @@ -65,10 +67,12 @@ int skiplist_set_init_ex(SkiplistSet *sl, const int level_count, } for (i=level_count-1; i>=0; i--) { + sprintf(name, "sl-set-level%02d", i); element_size = sizeof(SkiplistSetNode) + sizeof(SkiplistSetNode *) * (i + 1); - if ((result=fast_mblock_init_ex(sl->mblocks + i, - element_size, alloc_elements_once, NULL, NULL, false)) != 0) + if ((result=fast_mblock_init_ex1(sl->mblocks + i, name, + element_size, alloc_elements_once, alloc_elements_limit, + NULL, NULL, false)) != 0) { return result; } diff --git a/src/uniq_skiplist.c b/src/uniq_skiplist.c index 00c6784..a7e898b 100644 --- a/src/uniq_skiplist.c +++ b/src/uniq_skiplist.c @@ -49,6 +49,7 @@ int uniq_skiplist_init_ex2(UniqSkiplistFactory *factory, const int min_alloc_elements_once, const int delay_free_seconds, const bool bidirection) { + const int64_t alloc_elements_limit = 0; char name[64]; int bytes; int element_size; @@ -92,11 +93,12 @@ int uniq_skiplist_init_ex2(UniqSkiplistFactory *factory, extra_links_count = bidirection ? 1 : 0; for (i=max_level_count-1; i>=0; i--) { - sprintf(name, "sl-level%02d", i); + sprintf(name, "uniq-sl-level%02d", i); element_size = sizeof(UniqSkiplistNode) + sizeof(UniqSkiplistNode *) * (i + 1 + extra_links_count); if ((result=fast_mblock_init_ex1(factory->node_allocators + i, - name, element_size, alloc_elements_once, NULL, NULL, false)) != 0) + name, element_size, alloc_elements_once, + alloc_elements_limit, NULL, NULL, false)) != 0) { return result; } @@ -108,7 +110,7 @@ int uniq_skiplist_init_ex2(UniqSkiplistFactory *factory, if ((result=fast_mblock_init_ex1(&factory->skiplist_allocator, "skiplist", sizeof(UniqSkiplist), alloc_skiplist_once > 0 ? alloc_skiplist_once : - 16 * 1024, NULL, NULL, false)) != 0) + 16 * 1024, alloc_elements_limit, NULL, NULL, false)) != 0) { return result; }