From c39a4915dee7de6571604d77e2518ec116b53eaf Mon Sep 17 00:00:00 2001 From: Yu Qing Date: Fri, 6 Nov 2015 15:19:07 +0800 Subject: [PATCH] mblock support trunk malloc check and notify --- src/fast_mblock.c | 84 +++++++++++++++++++++++++++++++---------------- src/fast_mblock.h | 52 ++++++++++++++++++++--------- 2 files changed, 92 insertions(+), 44 deletions(-) diff --git a/src/fast_mblock.c b/src/fast_mblock.c index 254f4de..1c2ed18 100644 --- a/src/fast_mblock.c +++ b/src/fast_mblock.c @@ -100,7 +100,8 @@ static void delete_from_mblock_list(struct fast_mblock_man *mblock) #define STAT_DUP(pStat, current, copy_name) \ do { \ if (copy_name) { \ - strcpy(pStat->name, current->info.name); \ + strcpy(pStat->name, current->info.name); \ + pStat->trunk_size = current->info.trunk_size; \ pStat->element_size = current->info.element_size; \ } \ pStat->element_total_count += current->info.element_total_count; \ @@ -184,8 +185,8 @@ static int fast_mblock_info_cmp(const void *p1, const void *p2) pStat1 = (struct fast_mblock_info *)p1; pStat2 = (struct fast_mblock_info *)p2; - return GET_BLOCK_SIZE(*pStat2) * pStat2->element_total_count - - GET_BLOCK_SIZE(*pStat1) * pStat1->element_total_count; + return pStat2->trunk_size * pStat2->trunk_total_count - + pStat1->trunk_size * pStat1->trunk_total_count; } int fast_mblock_manager_stat_print(const bool hide_empty) @@ -220,24 +221,22 @@ int fast_mblock_manager_stat_print(const bool hide_empty) int64_t amem; char alloc_mem_str[32]; char used_mem_str[32]; - int block_size; qsort(stats, count, sizeof(struct fast_mblock_info), fast_mblock_info_cmp); alloc_mem = 0; used_mem = 0; - logInfo("%20s %12s %16s %12s %10s %10s %14s %12s %12s", "name", "element_size", - "instance_count", "alloc_bytes", "trunc_alloc", "trunk_used", + logInfo("%20s %12s %8s %12s %10s %10s %14s %12s %12s", "name", "element_size", + "instance", "alloc_bytes", "trunc_alloc", "trunk_used", "element_alloc", "element_used", "used_ratio"); stat_end = stats + count; for (pStat=stats; pStatelement_total_count > 0) + if (pStat->trunk_total_count > 0) { - block_size = GET_BLOCK_SIZE(*pStat); - amem = block_size * pStat->element_total_count; + amem = pStat->trunk_size * pStat->trunk_total_count; alloc_mem += amem; - used_mem += block_size * pStat->element_used_count; + used_mem += GET_BLOCK_SIZE(*pStat) * pStat->element_used_count; } else { @@ -248,7 +247,7 @@ int fast_mblock_manager_stat_print(const bool hide_empty) } } - logInfo("%20s %12d %16d %12d %10d %10d %14d %12d %11.2f%%", pStat->name, + logInfo("%20s %12d %8d %12d %10d %10d %14d %12d %11.2f%%", pStat->name, pStat->element_size, pStat->instance_count, amem, pStat->trunk_total_count, pStat->trunk_used_count, pStat->element_total_count, pStat->element_used_count, @@ -292,14 +291,18 @@ int fast_mblock_init_ex(struct fast_mblock_man *mblock, fast_mblock_alloc_init_func init_func, const bool need_lock) { return fast_mblock_init_ex2(mblock, NULL, element_size, - alloc_elements_once, init_func, need_lock); + alloc_elements_once, init_func, need_lock, NULL, NULL, NULL); } int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name, const int element_size, const int alloc_elements_once, - fast_mblock_alloc_init_func init_func, const bool need_lock) + fast_mblock_alloc_init_func init_func, const bool need_lock, + fast_mblock_malloc_trunk_check_func malloc_trunk_check, + fast_mblock_malloc_trunk_notify_func malloc_trunk_notify, + void *malloc_trunk_args) { int result; + int block_size; if (element_size <= 0) { @@ -310,14 +313,14 @@ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name, } mblock->info.element_size = MEM_ALIGN(element_size); + block_size = fast_mblock_get_block_size(mblock); if (alloc_elements_once > 0) { mblock->alloc_elements_once = alloc_elements_once; } else { - mblock->alloc_elements_once = (1024 * 1024) / - fast_mblock_get_block_size(mblock); + mblock->alloc_elements_once = (1024 * 1024) / block_size; } if (need_lock && (result=init_pthread_lock(&(mblock->lock))) != 0) @@ -332,13 +335,18 @@ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name, INIT_HEAD(&mblock->trunks.head); mblock->info.trunk_total_count = 0; mblock->info.trunk_used_count = 0; - mblock->free_chain_head = NULL; - mblock->delay_free_chain.head = NULL; - mblock->delay_free_chain.tail = NULL; + mblock->free_chain_head = NULL; + mblock->delay_free_chain.head = NULL; + mblock->delay_free_chain.tail = NULL; mblock->info.element_total_count = 0; mblock->info.element_used_count = 0; mblock->info.instance_count = 1; + mblock->info.trunk_size = sizeof(struct fast_mblock_malloc) + block_size * + mblock->alloc_elements_once; mblock->need_lock = need_lock; + mblock->malloc_trunk_callback.check_func = malloc_trunk_check; + mblock->malloc_trunk_callback.notify_func = malloc_trunk_notify; + mblock->malloc_trunk_callback.args = malloc_trunk_args; if (name != NULL) { @@ -350,7 +358,7 @@ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name, } add_to_mblock_list(mblock); - return 0; + return 0; } static int fast_mblock_prealloc(struct fast_mblock_man *mblock) @@ -361,29 +369,34 @@ static int fast_mblock_prealloc(struct fast_mblock_man *mblock) char *pTrunkStart; char *p; char *pLast; - int result; + int result; int block_size; - int alloc_size; block_size = fast_mblock_get_block_size(mblock); - alloc_size = sizeof(struct fast_mblock_malloc) + block_size * \ - mblock->alloc_elements_once; + if (mblock->malloc_trunk_callback.check_func != NULL && + mblock->malloc_trunk_callback.check_func( + mblock->info.trunk_size, + mblock->malloc_trunk_callback.args) != 0) + { + return ENOMEM; + } - pNew = (char *)malloc(alloc_size); + pNew = (char *)malloc(mblock->info.trunk_size); if (pNew == NULL) { logError("file: "__FILE__", line: %d, " \ "malloc %d bytes fail, " \ "errno: %d, error info: %s", \ - __LINE__, alloc_size, errno, STRERROR(errno)); + __LINE__, mblock->info.trunk_size, + errno, STRERROR(errno)); return errno != 0 ? errno : ENOMEM; } - memset(pNew, 0, alloc_size); + memset(pNew, 0, mblock->info.trunk_size); pMallocNode = (struct fast_mblock_malloc *)pNew; pTrunkStart = pNew + sizeof(struct fast_mblock_malloc); - pLast = pNew + (alloc_size - block_size); + pLast = pNew + (mblock->info.trunk_size - block_size); for (p=pTrunkStart; pinfo.trunk_total_count++; mblock->info.element_total_count += mblock->alloc_elements_once; - return 0; + + if (mblock->malloc_trunk_callback.notify_func != NULL) + { + mblock->malloc_trunk_callback.notify_func(mblock->info.trunk_size, + mblock->malloc_trunk_callback.args); + } + + return 0; } static inline void fast_mblock_remove_trunk(struct fast_mblock_man *mblock, @@ -431,6 +451,12 @@ static inline void fast_mblock_remove_trunk(struct fast_mblock_man *mblock, pMallocNode->next->prev = pMallocNode->prev; mblock->info.trunk_total_count--; mblock->info.element_total_count -= mblock->alloc_elements_once; + + if (mblock->malloc_trunk_callback.notify_func != NULL) + { + mblock->malloc_trunk_callback.notify_func(-1 * mblock->info.trunk_size, + mblock->malloc_trunk_callback.args); + } } #define FAST_MBLOCK_GET_TRUNK(pNode) \ @@ -488,7 +514,7 @@ void fast_mblock_destroy(struct fast_mblock_man *mblock) INIT_HEAD(&mblock->trunks.head); mblock->info.trunk_total_count = 0; mblock->info.trunk_used_count = 0; - mblock->free_chain_head = NULL; + mblock->free_chain_head = NULL; mblock->info.element_used_count = 0; mblock->info.element_total_count = 0; diff --git a/src/fast_mblock.h b/src/fast_mblock.h index 1ebaaf5..ee70e57 100644 --- a/src/fast_mblock.h +++ b/src/fast_mblock.h @@ -23,18 +23,18 @@ /* free node chain */ struct fast_mblock_node { - struct fast_mblock_node *next; + struct fast_mblock_node *next; int offset; //trunk offset int recycle_timestamp; - char data[0]; //the data buffer + char data[0]; //the data buffer }; /* malloc chain */ struct fast_mblock_malloc { int64_t ref_count; //refference count - struct fast_mblock_malloc *prev; - struct fast_mblock_malloc *next; + struct fast_mblock_malloc *prev; + struct fast_mblock_malloc *next; }; struct fast_mblock_chain { @@ -44,12 +44,19 @@ struct fast_mblock_chain { typedef int (*fast_mblock_alloc_init_func)(void *element); +typedef int (*fast_mblock_malloc_trunk_check_func)( + const int alloc_bytes, void *args); + +typedef void (*fast_mblock_malloc_trunk_notify_func)( + const int alloc_bytes, void *args); + struct fast_mblock_info { char name[FAST_MBLOCK_NAME_SIZE]; - int element_size; //element size + int element_size; //element size int element_total_count; //total element count int element_used_count; //used element count + int trunk_size; //trunk size int trunk_total_count; //total trunk count int trunk_used_count; //used trunk count int instance_count; //instance count @@ -60,17 +67,26 @@ struct fast_mblock_trunks struct fast_mblock_malloc head; //malloc chain to be freed }; +struct fast_mblock_malloc_trunk_callback +{ + fast_mblock_malloc_trunk_check_func check_func; + fast_mblock_malloc_trunk_notify_func notify_func; + void *args; +}; + struct fast_mblock_man { struct fast_mblock_info info; - int alloc_elements_once; //alloc elements once - struct fast_mblock_node *free_chain_head; //free node chain + int alloc_elements_once; //alloc elements once + struct fast_mblock_node *free_chain_head; //free node chain struct fast_mblock_trunks trunks; struct fast_mblock_chain delay_free_chain; //delay free node chain fast_mblock_alloc_init_func alloc_init_func; + struct fast_mblock_malloc_trunk_callback malloc_trunk_callback; + bool need_lock; //if need mutex lock - pthread_mutex_t lock; //the lock for read / write free node chain + pthread_mutex_t lock; //the lock for read / write free node chain struct fast_mblock_man *prev; //for stat manager struct fast_mblock_man *next; //for stat manager }; @@ -94,9 +110,9 @@ extern "C" { /** mblock init parameters: - mblock: the mblock pointer - element_size: element size, such as sizeof(struct xxx) - alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once + mblock: the mblock pointer + element_size: element size, such as sizeof(struct xxx) + alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once init_func: the init function need_lock: if need lock return error no, 0 for success, != 0 fail @@ -109,16 +125,22 @@ int fast_mblock_init_ex(struct fast_mblock_man *mblock, mblock init parameters: name: the mblock name - mblock: the mblock pointer - element_size: element size, such as sizeof(struct xxx) - alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once + mblock: the mblock pointer + element_size: element size, such as sizeof(struct xxx) + alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once init_func: the init function need_lock: if need lock + malloc_trunk_check: the malloc trunk check function pointor + malloc_trunk_notify: the malloc trunk notify function pointor + malloc_trunk__args: the malloc trunk args return error no, 0 for success, != 0 fail */ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name, const int element_size, const int alloc_elements_once, - fast_mblock_alloc_init_func init_func, const bool need_lock); + fast_mblock_alloc_init_func init_func, const bool need_lock, + fast_mblock_malloc_trunk_check_func malloc_trunk_check, + fast_mblock_malloc_trunk_notify_func malloc_trunk_notify, + void *malloc_trunk_args); /** mblock destroy