fast_mblock.[hc]: support alloc elements limit

HISTORY

@@ -1,5 +1,5 @@
 
-Version 1.44  2020-08-31
+Version 1.44  2020-09-01
   * add test file src/tests/test_pthread_lock.c
   * add uniq_skiplist.[hc]
   * add function split_string_ex
@@ -35,6 +35,7 @@ Version 1.44  2020-08-31
   * add files: shared_buffer.[hc]
   * add files: thread_pool.[hc]
   * shared_func.[hc]: add function fc_path_contains
+  * fast_mblock.[hc]: support alloc elements limit
 
 Version 1.43  2019-12-25
   * replace function call system to getExecResult,

src/common_blocked_queue.c

@@ -11,6 +11,7 @@
 int common_blocked_queue_init_ex(struct common_blocked_queue *queue,
         const int alloc_elements_once)
 {
+    const int64_t alloc_elements_limit = 0;
 	int result;
 
 	if ((result=init_pthread_lock(&queue->lock)) != 0)
@@ -30,10 +31,10 @@ int common_blocked_queue_init_ex(struct common_blocked_queue *queue,
         return result;
     }
 
-    if ((result=fast_mblock_init_ex2(&queue->mblock,
+    if ((result=fast_mblock_init_ex1(&queue->mblock, "queue_node",
-                    "queue_node", sizeof(struct common_blocked_node),
+                    sizeof(struct common_blocked_node),
-                    alloc_elements_once, NULL, NULL, false,
+                    alloc_elements_once, alloc_elements_limit,
-                    NULL, NULL, NULL)) != 0)
+                    NULL, NULL, false)) != 0)
     {
         return result;
     }

src/connection_pool.c

@@ -22,6 +22,7 @@ int conn_pool_init_ex1(ConnectionPool *cp, int connect_timeout,
     fc_connection_callback_func validate_func, void *validate_args,
     const int extra_data_size)
 {
+    const int64_t alloc_elements_limit = 0;
 	int result;
     int init_capacity;
 
@@ -40,15 +41,16 @@ int conn_pool_init_ex1(ConnectionPool *cp, int connect_timeout,
 
     init_capacity = htable_init_capacity > 0 ? htable_init_capacity : 256;
     if ((result=fast_mblock_init_ex1(&cp->manager_allocator, "cpool_manager",
-                    sizeof(ConnectionManager), init_capacity, NULL, NULL,
+                    sizeof(ConnectionManager), init_capacity,
-                    false)) != 0)
+                    alloc_elements_limit, NULL, NULL, false)) != 0)
     {
         return result;
     }
 
     if ((result=fast_mblock_init_ex1(&cp->node_allocator, "cpool_node",
                     sizeof(ConnectionNode) + sizeof(ConnectionInfo) +
-                    extra_data_size, init_capacity, NULL, NULL, false)) != 0)
+                    extra_data_size, init_capacity, alloc_elements_limit,
+                    NULL, NULL, false)) != 0)
     {
         return result;
     }

src/fast_allocator.c

@@ -128,6 +128,7 @@ static int allocator_array_check_capacity(struct fast_allocator_context *acontex
 static int region_init(struct fast_allocator_context *acontext,
         const char *mblock_name_prefix, struct fast_region_info *region)
 {
+    const int64_t alloc_elements_limit = 0;
 	int result;
 	int bytes;
 	int element_size;
@@ -167,8 +168,8 @@ static int region_init(struct fast_allocator_context *acontext,
             name = NULL;
         }
 		result = fast_mblock_init_ex2(&allocator->mblock, name, element_size,
-			region->alloc_elements_once, NULL, NULL, acontext->need_lock,
+			region->alloc_elements_once, alloc_elements_limit, NULL, NULL,
-			fast_allocator_malloc_trunk_check,
+            acontext->need_lock, fast_allocator_malloc_trunk_check,
 			fast_allocator_malloc_trunk_notify_func, acontext);
 		if (result != 0)
 		{

src/fast_mblock.c

@@ -23,6 +23,9 @@ struct _fast_mblock_manager
 
 static struct _fast_mblock_manager mblock_manager = {false, 0};
 
+#define fast_mblock_get_trunk_size(mblock, block_size, element_count) \
+    (sizeof(struct fast_mblock_malloc) + block_size * element_count)
+
 int fast_mblock_manager_init()
 {
     int result;
@@ -337,16 +340,18 @@ int fast_mblock_manager_stat_print_ex(const bool hide_empty, const int order_by)
 
 int fast_mblock_init_ex(struct fast_mblock_man *mblock,
         const int element_size, const int alloc_elements_once,
+        const int64_t alloc_elements_limit,
         fast_mblock_alloc_init_func init_func, void *init_args,
         const bool need_lock)
 {
     return fast_mblock_init_ex2(mblock, NULL, element_size,
-            alloc_elements_once, init_func, init_args,
+            alloc_elements_once, alloc_elements_limit, init_func,
-            need_lock, NULL, NULL, NULL);
+            init_args, need_lock, NULL, NULL, NULL);
 }
 
 int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name,
         const int element_size, const int alloc_elements_once,
+        const int64_t alloc_elements_limit,
         fast_mblock_alloc_init_func init_func,
         void *init_args, const bool need_lock,
         fast_mblock_malloc_trunk_check_func malloc_trunk_check,
@@ -365,14 +370,20 @@ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name,
 	}
 
 	mblock->info.element_size = MEM_ALIGN(element_size);
+    mblock->alloc_elements.limit = alloc_elements_limit;
 	block_size = fast_mblock_get_block_size(mblock);
 	if (alloc_elements_once > 0)
 	{
-		mblock->alloc_elements_once = alloc_elements_once;
+		mblock->alloc_elements.once = alloc_elements_once;
 	}
 	else
 	{
-		mblock->alloc_elements_once = (1024 * 1024) / block_size;
+		mblock->alloc_elements.once = (1024 * 1024) / block_size;
+	}
+    if (mblock->alloc_elements.limit > 0 && mblock->alloc_elements.once >
+            mblock->alloc_elements.limit)
+    {
+        mblock->alloc_elements.once = mblock->alloc_elements.limit;
     }
 
 	if (need_lock && (result=init_pthread_lock(&(mblock->lock))) != 0)
@@ -395,8 +406,8 @@ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name,
     mblock->info.element_total_count = 0;
     mblock->info.element_used_count = 0;
     mblock->info.instance_count = 1;
-    mblock->info.trunk_size = sizeof(struct fast_mblock_malloc) + block_size *
+    mblock->info.trunk_size = fast_mblock_get_trunk_size(mblock,
-			mblock->alloc_elements_once;
+            block_size, mblock->alloc_elements.once);
     mblock->need_lock = need_lock;
     mblock->malloc_trunk_callback.check_func = malloc_trunk_check;
     mblock->malloc_trunk_callback.notify_func = malloc_trunk_notify;
@@ -425,27 +436,52 @@ static int fast_mblock_prealloc(struct fast_mblock_man *mblock)
 	char *pLast;
 	int result;
 	int block_size;
+    int trunk_size;
+    int alloc_count;
 
 	block_size = fast_mblock_get_block_size(mblock);
+    if (mblock->alloc_elements.limit > 0)
+    {
+        int64_t avail_count;
+        avail_count = mblock->alloc_elements.limit -
+            mblock->info.element_total_count;
+        if (avail_count <= 0)
+        {
+            logError("file: "__FILE__", line: %d, "
+                    "allocated elements exceed limit: %"PRId64,
+                    __LINE__, mblock->alloc_elements.limit);
+            return EOVERFLOW;
+        }
+
+        alloc_count = avail_count > mblock->alloc_elements.once ?
+            mblock->alloc_elements.once : avail_count;
+        trunk_size = fast_mblock_get_trunk_size(mblock,
+                block_size, alloc_count);
+    }
+    else
+    {
+        alloc_count = mblock->alloc_elements.once;
+        trunk_size = mblock->info.trunk_size;
+    }
+
 	if (mblock->malloc_trunk_callback.check_func != NULL &&
-		mblock->malloc_trunk_callback.check_func(
+		mblock->malloc_trunk_callback.check_func(trunk_size,
-		mblock->info.trunk_size,
             mblock->malloc_trunk_callback.args) != 0)
 	{
 		return ENOMEM;
 	}
 
-	pNew = (char *)fc_malloc(mblock->info.trunk_size);
+	pNew = (char *)fc_malloc(trunk_size);
 	if (pNew == NULL)
 	{
 		return ENOMEM;
 	}
-	memset(pNew, 0, mblock->info.trunk_size);
+	memset(pNew, 0, trunk_size);
 
 	pMallocNode = (struct fast_mblock_malloc *)pNew;
 
 	pTrunkStart = pNew + sizeof(struct fast_mblock_malloc);
-	pLast = pNew + (mblock->info.trunk_size - block_size);
+	pLast = pNew + (trunk_size - block_size);
 	for (p=pTrunkStart; p<=pLast; p += block_size)
 	{
 		pNode = (struct fast_mblock_node *)p;
@@ -466,17 +502,19 @@ static int fast_mblock_prealloc(struct fast_mblock_man *mblock)
 	mblock->free_chain_head = (struct fast_mblock_node *)pTrunkStart;
 
     pMallocNode->ref_count = 0;
+    pMallocNode->alloc_count = alloc_count;
+    pMallocNode->trunk_size = trunk_size;
     pMallocNode->prev = mblock->trunks.head.prev;
 	pMallocNode->next = &mblock->trunks.head;
     mblock->trunks.head.prev->next = pMallocNode;
     mblock->trunks.head.prev = pMallocNode;
 
     mblock->info.trunk_total_count++;
-    mblock->info.element_total_count += mblock->alloc_elements_once;
+    mblock->info.element_total_count += alloc_count;
 
     if (mblock->malloc_trunk_callback.notify_func != NULL)
     {
-	   mblock->malloc_trunk_callback.notify_func(mblock->info.trunk_size,
+        mblock->malloc_trunk_callback.notify_func(trunk_size,
                 mblock->malloc_trunk_callback.args);
     }
 
@@ -489,11 +527,11 @@ static inline void fast_mblock_remove_trunk(struct fast_mblock_man *mblock,
     pMallocNode->prev->next = pMallocNode->next;
 	pMallocNode->next->prev = pMallocNode->prev;
     mblock->info.trunk_total_count--;
-    mblock->info.element_total_count -= mblock->alloc_elements_once;
+    mblock->info.element_total_count -= pMallocNode->alloc_count;
 
     if (mblock->malloc_trunk_callback.notify_func != NULL)
     {
-	   mblock->malloc_trunk_callback.notify_func(-1 * mblock->info.trunk_size,
+	   mblock->malloc_trunk_callback.notify_func(-1 * pMallocNode->trunk_size,
 	   mblock->malloc_trunk_callback.args);
     }
 }
@@ -570,9 +608,9 @@ struct fast_mblock_node *fast_mblock_alloc(struct fast_mblock_man *mblock)
 
 	if (mblock->need_lock && (result=pthread_mutex_lock(&(mblock->lock))) != 0)
 	{
-		logError("file: "__FILE__", line: %d, " \
+		logError("file: "__FILE__", line: %d, "
-			"call pthread_mutex_lock fail, " \
+			"call pthread_mutex_lock fail, "
-			"errno: %d, error info: %s", \
+			"errno: %d, error info: %s",
 			__LINE__, result, STRERROR(result));
 		return NULL;
 	}

src/fast_mblock.h

@@ -37,6 +37,8 @@ struct fast_mblock_node
 struct fast_mblock_malloc
 {
     int64_t ref_count; //refference count
+    int alloc_count;   //allocated element count
+    int trunk_size;    //trunk bytes
     struct fast_mblock_malloc *prev;
     struct fast_mblock_malloc *next;
 };
@@ -82,7 +84,10 @@ struct fast_mblock_malloc_trunk_callback
 struct fast_mblock_man
 {
     struct fast_mblock_info info;
-    int alloc_elements_once;  //alloc elements once
+    struct {
+        int once;        //alloc elements once
+        int64_t limit;   //<= 0 for no limit
+    } alloc_elements;
     struct fast_mblock_node *free_chain_head;    //free node chain
     struct fast_mblock_trunks trunks;
     struct fast_mblock_chain delay_free_chain;   //delay free node chain
@@ -112,7 +117,7 @@ extern "C" {
 
 #define fast_mblock_init(mblock, element_size, alloc_elements_once) \
     fast_mblock_init_ex(mblock, element_size, alloc_elements_once,  \
-            NULL, NULL, true)
+            0, NULL, NULL, true)
 
 /**
 mblock init
@@ -120,6 +125,7 @@ parameters:
     mblock: the mblock pointer
     element_size: element size, such as sizeof(struct xxx)
     alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once
+    alloc_elements_limit: malloc elements limit, <= 0 for no limit
     init_func: the init function
     init_args: the args for init_func
     need_lock: if need lock
@@ -127,6 +133,7 @@ return error no, 0 for success, != 0 fail
 */
 int fast_mblock_init_ex(struct fast_mblock_man *mblock,
         const int element_size, const int alloc_elements_once,
+        const int64_t alloc_elements_limit,
         fast_mblock_alloc_init_func init_func, void *init_args,
         const bool need_lock);
 
@@ -137,6 +144,7 @@ parameters:
     mblock: the mblock pointer
     element_size: element size, such as sizeof(struct xxx)
     alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once
+    alloc_elements_limit: malloc elements limit, <= 0 for no limit
     init_func: the init function
     init_args: the args for init_func
     need_lock: if need lock
@@ -147,6 +155,7 @@ return error no, 0 for success, != 0 fail
 */
 int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name,
         const int element_size, const int alloc_elements_once,
+        const int64_t alloc_elements_limit,
         fast_mblock_alloc_init_func init_func,
         void *init_args, const bool need_lock,
         fast_mblock_malloc_trunk_check_func malloc_trunk_check,
@@ -160,6 +169,7 @@ parameters:
     mblock: the mblock pointer
     element_size: element size, such as sizeof(struct xxx)
     alloc_elements_once: malloc elements once, 0 for malloc 1MB memory once
+    alloc_elements_limit: malloc elements limit, <= 0 for no limit
     init_func: the init function
     init_args: the args for init_func
     need_lock: if need lock
@@ -168,12 +178,13 @@ return error no, 0 for success, != 0 fail
 static inline int fast_mblock_init_ex1(struct fast_mblock_man *mblock,
         const char *name, const int element_size,
         const int alloc_elements_once,
+        const int64_t alloc_elements_limit,
         fast_mblock_alloc_init_func init_func,
         void *init_args, const bool need_lock)
 {
     return fast_mblock_init_ex2(mblock, name, element_size,
-            alloc_elements_once, init_func, init_args,
+            alloc_elements_once, alloc_elements_limit, init_func,
-            need_lock, NULL, NULL, NULL);
+            init_args, need_lock, NULL, NULL, NULL);
 }
 
 /**

src/flat_skiplist.c

@@ -22,6 +22,8 @@ int flat_skiplist_init_ex(FlatSkiplist *sl, const int level_count,
         skiplist_compare_func compare_func, skiplist_free_func free_func,
         const int min_alloc_elements_once)
 {
+    const int64_t alloc_elements_limit = 0;
+    char name[64];
     int bytes;
     int element_size;
     int i;
@@ -65,10 +67,12 @@ int flat_skiplist_init_ex(FlatSkiplist *sl, const int level_count,
     }
 
     for (i=level_count-1; i>=0; i--) {
+        sprintf(name, "flat-sl-level%02d", i);
         element_size = sizeof(FlatSkiplistNode) +
             sizeof(FlatSkiplistNode *) * (i + 1);
-        if ((result=fast_mblock_init_ex(sl->mblocks + i,
+        if ((result=fast_mblock_init_ex1(sl->mblocks + i, name,
-            element_size, alloc_elements_once, NULL, NULL, false)) != 0)
+            element_size, alloc_elements_once, alloc_elements_limit,
+            NULL, NULL, false)) != 0)
         {
             return result;
         }

src/multi_skiplist.c

@@ -23,6 +23,8 @@ int multi_skiplist_init_ex(MultiSkiplist *sl, const int level_count,
         skiplist_free_func free_func,
         const int min_alloc_elements_once)
 {
+    const int64_t alloc_elements_limit = 0;
+    char name[64];
     int bytes;
     int element_size;
     int i;
@@ -66,10 +68,12 @@ int multi_skiplist_init_ex(MultiSkiplist *sl, const int level_count,
     }
 
     for (i=level_count-1; i>=0; i--) {
+        sprintf(name, "multi-sl-level%02d", i);
         element_size = sizeof(MultiSkiplistNode) +
             sizeof(MultiSkiplistNode *) * (i + 1);
-        if ((result=fast_mblock_init_ex(sl->mblocks + i,
+        if ((result=fast_mblock_init_ex1(sl->mblocks + i, name,
-            element_size, alloc_elements_once, NULL, NULL, false)) != 0)
+            element_size, alloc_elements_once, alloc_elements_limit,
+            NULL, NULL, false)) != 0)
         {
             return result;
         }
@@ -92,9 +96,9 @@ int multi_skiplist_init_ex(MultiSkiplist *sl, const int level_count,
     }
     memset(sl->tail, 0, sl->mblocks[0].info.element_size);
 
-    if ((result=fast_mblock_init_ex(&sl->data_mblock,
+    if ((result=fast_mblock_init_ex1(&sl->data_mblock, "multi-sl-data",
                     sizeof(MultiSkiplistData), alloc_elements_once,
-                    NULL, NULL, false)) != 0)
+                    alloc_elements_limit, NULL, NULL, false)) != 0)
     {
         return result;
     }

src/shared_buffer.c

@@ -23,13 +23,14 @@ int shared_buffer_init_ex(SharedBufferContext *context,
         const int alloc_elements_once, const int buffer_init_capacity,
         const bool need_lock)
 {
+    const int64_t alloc_elements_limit = 0;
     int result;
 
     context->buffer_init_capacity = buffer_init_capacity;
-    if ((result=fast_mblock_init_ex2(&context->allocator, "shared_buffer",
+    if ((result=fast_mblock_init_ex1(&context->allocator, "shared_buffer",
                     sizeof(SharedBuffer), alloc_elements_once,
-                    shared_buffer_alloc_init, context, need_lock,
+                    alloc_elements_limit, shared_buffer_alloc_init,
-                    NULL, NULL, NULL)) != 0)
+                    context, need_lock)) != 0)
     {
         return result;
     }

src/skiplist_set.c

@@ -22,6 +22,8 @@ int skiplist_set_init_ex(SkiplistSet *sl, const int level_count,
         skiplist_compare_func compare_func, skiplist_free_func free_func,
         const int min_alloc_elements_once)
 {
+    const int64_t alloc_elements_limit = 0;
+    char name[64];
     int bytes;
     int element_size;
     int i;
@@ -65,10 +67,12 @@ int skiplist_set_init_ex(SkiplistSet *sl, const int level_count,
     }
 
     for (i=level_count-1; i>=0; i--) {
+        sprintf(name, "sl-set-level%02d", i);
         element_size = sizeof(SkiplistSetNode) +
             sizeof(SkiplistSetNode *) * (i + 1);
-        if ((result=fast_mblock_init_ex(sl->mblocks + i,
+        if ((result=fast_mblock_init_ex1(sl->mblocks + i, name,
-            element_size, alloc_elements_once, NULL, NULL, false)) != 0)
+            element_size, alloc_elements_once, alloc_elements_limit,
+            NULL, NULL, false)) != 0)
         {
             return result;
         }

src/uniq_skiplist.c

@@ -49,6 +49,7 @@ int uniq_skiplist_init_ex2(UniqSkiplistFactory *factory,
         const int min_alloc_elements_once, const int delay_free_seconds,
         const bool bidirection)
 {
+    const int64_t alloc_elements_limit = 0;
     char name[64];
     int bytes;
     int element_size;
@@ -92,11 +93,12 @@ int uniq_skiplist_init_ex2(UniqSkiplistFactory *factory,
 
     extra_links_count = bidirection ? 1 : 0;
     for (i=max_level_count-1; i>=0; i--) {
-        sprintf(name, "sl-level%02d", i);
+        sprintf(name, "uniq-sl-level%02d", i);
         element_size = sizeof(UniqSkiplistNode) +
             sizeof(UniqSkiplistNode *) * (i + 1 + extra_links_count);
         if ((result=fast_mblock_init_ex1(factory->node_allocators + i,
-            name, element_size, alloc_elements_once, NULL, NULL, false)) != 0)
+                        name, element_size, alloc_elements_once,
+                        alloc_elements_limit, NULL, NULL, false)) != 0)
         {
             return result;
         }
@@ -108,7 +110,7 @@ int uniq_skiplist_init_ex2(UniqSkiplistFactory *factory,
     if ((result=fast_mblock_init_ex1(&factory->skiplist_allocator,
                     "skiplist", sizeof(UniqSkiplist),
                     alloc_skiplist_once > 0 ?  alloc_skiplist_once :
-                    16 * 1024, NULL, NULL, false)) != 0)
+                    16 * 1024, alloc_elements_limit, NULL, NULL, false)) != 0)
     {
         return result;
     }