pdxx
/
libfastcommon
mirror of https://github.com/happyfish100/libfastcommon.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fast_mblock stats refine

pull/37/head
YuQing 2020-03-29 12:30:42 +08:00
parent 590aa8d3a0
commit 427818d005
9 changed files with 122 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
HISTORY
View File

@ -1,5 +1,5 @@
Version 1.44 2020-03-28
Version 1.44 2020-03-29
* add test file src/tests/test_pthread_lock.c
* add uniq_skiplist.[hc]
* add function split_string_ex

2
src/common_define.h
View File

@ -116,6 +116,8 @@ extern int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int kind);
#define ENONET 64 /* Machine is not on the network */
#endif
#define compile_barrier() __asm__ __volatile__("" : : : "memory")
#define IS_UPPER_HEX(ch) ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F'))
#define IS_HEX_CHAR(ch) (IS_UPPER_HEX(ch) || (ch >= 'a' && ch <= 'f'))
#define FC_IS_DIGITAL(ch) (ch >= '0' && ch <= '9')

36
src/fast_allocator.c
View File

@ -108,13 +108,15 @@ static int allocator_array_check_capacity(struct fast_allocator_context *acontex
}
static int region_init(struct fast_allocator_context *acontext,
struct fast_region_info *region)
const char *mblock_name_prefix, struct fast_region_info *region)
{
int result;
int bytes;
int element_size;
int allocator_count;
struct fast_allocator_info *allocator;
char *name;
char name_buff[FAST_MBLOCK_NAME_SIZE];
region->pad_mask = region->step - 1;
allocator_count = (region->end - region->start) / region->step;
@ -136,12 +138,22 @@ static int region_init(struct fast_allocator_context *acontext,
return result;
}
name = name_buff;
result = 0;
allocator = region->allocators;
for (element_size=region->start+region->step; element_size<=region->end;
element_size+=region->step,allocator++)
{
result = fast_mblock_init_ex2(&allocator->mblock, NULL, element_size,
if (mblock_name_prefix != NULL)
{
snprintf(name, FAST_MBLOCK_NAME_SIZE, "%s-%d",
mblock_name_prefix, element_size);
}
else
{
name = NULL;
}
result = fast_mblock_init_ex2(&allocator->mblock, name, element_size,
region->alloc_elements_once, NULL, NULL, acontext->need_lock,
fast_allocator_malloc_trunk_check,
fast_allocator_malloc_trunk_notify_func, acontext);
@ -174,9 +186,10 @@ static void region_destroy(struct fast_allocator_context *acontext,
}
int fast_allocator_init_ex(struct fast_allocator_context *acontext,
struct fast_region_info *regions, const int region_count,
const int64_t alloc_bytes_limit, const double expect_usage_ratio,
const int reclaim_interval, const bool need_lock)
const char *mblock_name_prefix, struct fast_region_info *regions,
const int region_count, const int64_t alloc_bytes_limit,
const double expect_usage_ratio, const int reclaim_interval,
const bool need_lock)
{
int result;
int bytes;
@ -263,7 +276,7 @@ int fast_allocator_init_ex(struct fast_allocator_context *acontext,
}
previous_end = pRegion->end;
if ((result=region_init(acontext, pRegion)) != 0)
if ((result=region_init(acontext, mblock_name_prefix, pRegion)) != 0)
{
break;
}
@ -288,8 +301,9 @@ int fast_allocator_init_ex(struct fast_allocator_context *acontext,
}
int fast_allocator_init(struct fast_allocator_context *acontext,
const int64_t alloc_bytes_limit, const double expect_usage_ratio,
const int reclaim_interval, const bool need_lock)
const char *mblock_name_prefix, const int64_t alloc_bytes_limit,
const double expect_usage_ratio, const int reclaim_interval,
const bool need_lock)
{
#define DEFAULT_REGION_COUNT 5
@ -301,9 +315,9 @@ int fast_allocator_init(struct fast_allocator_context *acontext,
FAST_ALLOCATOR_INIT_REGION(regions[3], 4096, 16384, 256, 64);
FAST_ALLOCATOR_INIT_REGION(regions[4], 16384, 65536, 1024, 16);
return fast_allocator_init_ex(acontext, regions,
DEFAULT_REGION_COUNT, alloc_bytes_limit,
expect_usage_ratio, reclaim_interval, need_lock);
return fast_allocator_init_ex(acontext, mblock_name_prefix, regions,
DEFAULT_REGION_COUNT, alloc_bytes_limit, expect_usage_ratio,
reclaim_interval, need_lock);
}
void fast_allocator_destroy(struct fast_allocator_context *acontext)

15
src/fast_allocator.h
View File

@ -76,15 +76,17 @@ extern "C" {
allocator init by default region allocators
parameters:
acontext: the context pointer
alloc_bytes_limit: the alloc limit, 0 for no limit
mblock_name_prefix: the name prefix of mblock
alloc_bytes_limit: the alloc limit, 0 for no limit
expect_usage_ratio: the trunk usage ratio
reclaim_interval: reclaim interval in second, 0 for never reclaim
need_lock: if need lock
return error no, 0 for success, != 0 fail
*/
int fast_allocator_init(struct fast_allocator_context *acontext,
const int64_t alloc_bytes_limit, const double expect_usage_ratio,
const int reclaim_interval, const bool need_lock);
const char *mblock_name_prefix, const int64_t alloc_bytes_limit,
const double expect_usage_ratio, const int reclaim_interval,
const bool need_lock);
/**
allocator init
@ -99,9 +101,10 @@ parameters:
return error no, 0 for success, != 0 fail
*/
int fast_allocator_init_ex(struct fast_allocator_context *acontext,
struct fast_region_info *regions, const int region_count,
const int64_t alloc_bytes_limit, const double expect_usage_ratio,
const int reclaim_interval, const bool need_lock);
const char *mblock_name_prefix, struct fast_region_info *regions,
const int region_count, const int64_t alloc_bytes_limit,
const double expect_usage_ratio, const int reclaim_interval,
const bool need_lock);
/**
allocator destroy

73
src/fast_mblock.c
View File

@ -109,6 +109,7 @@ static void delete_from_mblock_list(struct fast_mblock_man *mblock)
} \
pStat->element_total_count += current->info.element_total_count; \
pStat->element_used_count += current->info.element_used_count; \
pStat->delay_free_elements += current->info.delay_free_elements; \
pStat->trunk_total_count += current->info.trunk_total_count; \
pStat->trunk_used_count += current->info.trunk_used_count; \
pStat->instance_count += current->info.instance_count; \
@ -233,8 +234,10 @@ int fast_mblock_manager_stat_print_ex(const bool hide_empty, const int order_by)
int64_t alloc_mem;
int64_t used_mem;
int64_t amem;
int64_t delay_free_mem;
char alloc_mem_str[32];
char used_mem_str[32];
char delay_free_mem_str[32];
if (order_by == FAST_MBLOCK_ORDER_BY_ALLOC_BYTES)
{
@ -249,31 +252,37 @@ int fast_mblock_manager_stat_print_ex(const bool hide_empty, const int order_by)
alloc_mem = 0;
used_mem = 0;
logInfo("%20s %12s %8s %12s %10s %10s %14s %12s %12s", "name", "element_size",
"instance", "alloc_bytes", "trunc_alloc", "trunk_used",
"element_alloc", "element_used", "used_ratio");
delay_free_mem = 0;
logInfo("%20s %8s %8s %12s %10s %10s %10s %10s %10s %12s",
"name", "el_size", "instance", "alloc_bytes",
"trunc_alloc", "trunk_used", "el_alloc",
"el_used", "delay_free", "used_ratio");
stat_end = stats + count;
for (pStat=stats; pStat<stat_end; pStat++)
{
if (pStat->trunk_total_count > 0)
{
amem = pStat->trunk_size * pStat->trunk_total_count;
amem = pStat->trunk_size * pStat->trunk_total_count;
alloc_mem += amem;
used_mem += GET_BLOCK_SIZE(*pStat) * pStat->element_used_count;
used_mem += GET_BLOCK_SIZE(*pStat) *
pStat->element_used_count;
delay_free_mem += GET_BLOCK_SIZE(*pStat) *
pStat->delay_free_elements;
}
else
{
amem = 0;
amem = 0;
if (hide_empty)
{
continue;
}
}
logInfo("%20s %12d %8d %12"PRId64" %10d %10d %14d %12d %11.2f%%", pStat->name,
pStat->element_size, pStat->instance_count, amem,
pStat->trunk_total_count, pStat->trunk_used_count,
logInfo("%20s %8d %8d %12"PRId64" %10d %10d %10d %10d %10d %11.2f%%",
pStat->name, pStat->element_size, pStat->instance_count,
amem, pStat->trunk_total_count, pStat->trunk_used_count,
pStat->element_total_count, pStat->element_used_count,
pStat->delay_free_elements,
pStat->element_total_count > 0 ? 100.00 * (double)
pStat->element_used_count / (double)
pStat->element_total_count : 0.00);
@ -283,26 +292,40 @@ int fast_mblock_manager_stat_print_ex(const bool hide_empty, const int order_by)
{
sprintf(alloc_mem_str, "%"PRId64" bytes", alloc_mem);
sprintf(used_mem_str, "%"PRId64" bytes", used_mem);
sprintf(delay_free_mem_str, "%"PRId64" bytes", delay_free_mem);
}
else if (alloc_mem < 1024 * 1024)
{
sprintf(alloc_mem_str, "%.3f KB", (double)alloc_mem / 1024);
sprintf(used_mem_str, "%.3f KB", (double)used_mem / 1024);
sprintf(delay_free_mem_str, "%.3f KB",
(double)delay_free_mem / 1024);
}
else if (alloc_mem < 1024 * 1024 * 1024)
{
sprintf(alloc_mem_str, "%.3f MB", (double)alloc_mem / (1024 * 1024));
sprintf(used_mem_str, "%.3f MB", (double)used_mem / (1024 * 1024));
sprintf(alloc_mem_str, "%.3f MB",
(double)alloc_mem / (1024 * 1024));
sprintf(used_mem_str, "%.3f MB",
(double)used_mem / (1024 * 1024));
sprintf(delay_free_mem_str, "%.3f MB",
(double)delay_free_mem / (1024 * 1024));
}
else
{
sprintf(alloc_mem_str, "%.3f GB", (double)alloc_mem / (1024 * 1024 * 1024));
sprintf(used_mem_str, "%.3f GB", (double)used_mem / (1024 * 1024 * 1024));
sprintf(alloc_mem_str, "%.3f GB",
(double)alloc_mem / (1024 * 1024 * 1024));
sprintf(used_mem_str, "%.3f GB",
(double)used_mem / (1024 * 1024 * 1024));
sprintf(delay_free_mem_str, "%.3f GB",
(double)delay_free_mem / (1024 * 1024 * 1024));
}
logInfo("mblock entry count: %d, alloc memory: %s, used memory: %s, used ratio: %.2f%%",
count, alloc_mem_str, used_mem_str,
alloc_mem > 0 ? 100.00 * (double)used_mem / alloc_mem : 0.00);
logInfo("mblock entry count: %d, memory stat => { alloc : %s, "
"used: %s (%.2f%%), delay free: %s (%.2f%%) }",
count, alloc_mem_str, used_mem_str, alloc_mem > 0 ?
100.00 * (double)used_mem / alloc_mem : 0.00,
delay_free_mem_str, alloc_mem > 0 ? 100.00 *
(double)delay_free_mem / alloc_mem : 0.00);
}
if (stats != NULL) free(stats);
@ -362,6 +385,7 @@ int fast_mblock_init_ex2(struct fast_mblock_man *mblock, const char *name,
INIT_HEAD(&mblock->trunks.head);
mblock->info.trunk_total_count = 0;
mblock->info.trunk_used_count = 0;
mblock->info.delay_free_elements = 0;
mblock->free_chain_head = NULL;
mblock->delay_free_chain.head = NULL;
mblock->delay_free_chain.tail = NULL;
@ -534,6 +558,7 @@ void fast_mblock_destroy(struct fast_mblock_man *mblock)
mblock->info.trunk_used_count = 0;
mblock->free_chain_head = NULL;
mblock->info.element_used_count = 0;
mblock->info.delay_free_elements = 0;
mblock->info.element_total_count = 0;
if (mblock->need_lock) pthread_mutex_destroy(&(mblock->lock));
@ -558,9 +583,6 @@ struct fast_mblock_node *fast_mblock_alloc(struct fast_mblock_man *mblock)
{
pNode = mblock->free_chain_head;
mblock->free_chain_head = pNode->next;
mblock->info.element_used_count++;
fast_mblock_ref_counter_inc(mblock, pNode);
}
else
{
@ -573,13 +595,13 @@ struct fast_mblock_node *fast_mblock_alloc(struct fast_mblock_man *mblock)
{
mblock->delay_free_chain.tail = NULL;
}
mblock->info.delay_free_elements--;
}
else if ((result=fast_mblock_prealloc(mblock)) == 0)
{
pNode = mblock->free_chain_head;
mblock->free_chain_head = pNode->next;
mblock->info.element_used_count++;
fast_mblock_ref_counter_inc(mblock, pNode);
}
else
{
@ -587,6 +609,11 @@ struct fast_mblock_node *fast_mblock_alloc(struct fast_mblock_man *mblock)
}
}
if (pNode != NULL)
{
mblock->info.element_used_count++;
fast_mblock_ref_counter_inc(mblock, pNode);
}
if (mblock->need_lock && (result=pthread_mutex_unlock(&(mblock->lock))) != 0)
{
logError("file: "__FILE__", line: %d, " \
@ -654,6 +681,10 @@ int fast_mblock_delay_free(struct fast_mblock_man *mblock,
mblock->delay_free_chain.tail = pNode;
pNode->next = NULL;
mblock->info.element_used_count--;
mblock->info.delay_free_elements++;
fast_mblock_ref_counter_dec(mblock, pNode);
if (mblock->need_lock && (result=pthread_mutex_unlock(&(mblock->lock))) != 0)
{
logError("file: "__FILE__", line: %d, " \

1
src/fast_mblock.h
View File

@ -59,6 +59,7 @@ struct fast_mblock_info
int element_size; //element size
int element_total_count; //total element count
int element_used_count; //used element count
int delay_free_elements; //delay free element count
int trunk_size; //trunk size
int trunk_total_count; //total trunk count
int trunk_used_count; //used trunk count

2
src/tests/test_allocator.c
View File

@ -43,7 +43,7 @@ int main(int argc, char *argv[])
g_log_context.log_level = LOG_DEBUG;
fast_mblock_manager_init();
if ((result=fast_allocator_init(&acontext, 0, 0.00, 0, true)) != 0)
if ((result=fast_allocator_init(&acontext, NULL, 0, 0.00, 0, true)) != 0)
{
return result;
}

59
src/uniq_skiplist.c
View File

@ -112,7 +112,14 @@ int uniq_skiplist_init_ex(UniqSkiplistFactory *factory,
{
return result;
}
factory->tail = (UniqSkiplistNode *)fast_mblock_alloc_object(
factory->node_allocators + 0);
if (factory->tail == NULL) {
return ENOMEM;
}
memset(factory->tail, 0, factory->node_allocators[0].info.element_size);
factory->max_level_count = max_level_count;
factory->compare_func = compare_func;
factory->free_func = free_func;
@ -130,6 +137,8 @@ void uniq_skiplist_destroy(UniqSkiplistFactory *factory)
return;
}
fast_mblock_free_object(factory->node_allocators + 0,
(void *)factory->tail);
fast_mblock_destroy(&factory->skiplist_allocator);
for (i=0; i<factory->max_level_count; i++) {
@ -178,16 +187,8 @@ UniqSkiplist *uniq_skiplist_new(UniqSkiplistFactory *factory,
}
memset(sl->top, 0, top_mblock->info.element_size);
sl->tail = (UniqSkiplistNode *)fast_mblock_alloc_object(
sl->factory->node_allocators + 0);
if (sl->tail == NULL) {
errno = ENOMEM;
return NULL;
}
memset(sl->tail, 0, sl->factory->node_allocators[0].info.element_size);
for (i=0; i<level_count; i++) {
sl->top->links[i] = sl->tail;
sl->top->links[i] = sl->factory->tail;
}
return sl;
@ -214,7 +215,7 @@ static int uniq_skiplist_grow(UniqSkiplist *sl)
}
memset(top, 0, top_mblock->info.element_size);
top->links[top_level_index] = sl->tail;
top->links[top_level_index] = sl->factory->tail;
for (i=0; i<=sl->top_level_index; i++) {
top->links[i] = sl->top->links[i];
}
@ -223,7 +224,7 @@ static int uniq_skiplist_grow(UniqSkiplist *sl)
old_top = sl->top;
sl->top = top;
__sync_synchronize();
compile_barrier();
sl->top_level_index = top_level_index;
UNIQ_SKIPLIST_FREE_MBLOCK_OBJECT(sl, old_top_level_index, old_top);
@ -241,7 +242,7 @@ void uniq_skiplist_free(UniqSkiplist *sl)
node = sl->top->links[0];
if (sl->factory->free_func != NULL) {
while (node != sl->tail) {
while (node != sl->factory->tail) {
deleted = node;
node = node->links[0];
sl->factory->free_func(deleted->data, 0);
@ -249,7 +250,7 @@ void uniq_skiplist_free(UniqSkiplist *sl)
deleted->level_index, (void *)deleted);
}
} else {
while (node != sl->tail) {
while (node != sl->factory->tail) {
deleted = node;
node = node->links[0];
fast_mblock_free_object(sl->factory->node_allocators +
@ -259,11 +260,8 @@ void uniq_skiplist_free(UniqSkiplist *sl)
fast_mblock_free_object(sl->factory->node_allocators +
sl->top_level_index, (void *)sl->top);
fast_mblock_free_object(sl->factory->node_allocators + 0,
(void *)sl->tail);
sl->top = NULL;
sl->tail = NULL;
sl->element_count = 0;
fast_mblock_free_object(&sl->factory->skiplist_allocator, sl);
}
@ -293,7 +291,7 @@ int uniq_skiplist_insert(UniqSkiplist *sl, void *data)
level_index = uniq_skiplist_get_level_index(sl);
previous = sl->top;
for (i=sl->top_level_index; i>level_index; i--) {
while (previous->links[i] != sl->tail) {
while (previous->links[i] != sl->factory->tail) {
cmp = sl->factory->compare_func(data, previous->links[i]->data);
if (cmp < 0) {
break;
@ -307,7 +305,7 @@ int uniq_skiplist_insert(UniqSkiplist *sl, void *data)
}
while (i >= 0) {
while (previous->links[i] != sl->tail) {
while (previous->links[i] != sl->factory->tail) {
cmp = sl->factory->compare_func(data, previous->links[i]->data);
if (cmp < 0) {
break;
@ -331,7 +329,7 @@ int uniq_skiplist_insert(UniqSkiplist *sl, void *data)
node->level_index = level_index;
node->data = data;
__sync_synchronize();
compile_barrier();
//thread safe for one write with many read model
for (i=0; i<=level_index; i++) {
@ -356,7 +354,7 @@ static UniqSkiplistNode *uniq_skiplist_get_equal_previous(UniqSkiplist *sl,
previous = sl->top;
for (i=sl->top_level_index; i>=0; i--) {
while (previous->links[i] != sl->tail) {
while (previous->links[i] != sl->factory->tail) {
cmp = sl->factory->compare_func(data, previous->links[i]->data);
if (cmp < 0) {
break;
@ -382,7 +380,7 @@ static UniqSkiplistNode *uniq_skiplist_get_first_larger_or_equal(
previous = sl->top;
for (i=sl->top_level_index; i>=0; i--) {
while (previous->links[i] != sl->tail) {
while (previous->links[i] != sl->factory->tail) {
cmp = sl->factory->compare_func(data, previous->links[i]->data);
if (cmp < 0) {
break;
@ -407,7 +405,7 @@ static UniqSkiplistNode *uniq_skiplist_get_first_larger(
previous = sl->top;
for (i=sl->top_level_index; i>=0; i--) {
while (previous->links[i] != sl->tail) {
while (previous->links[i] != sl->factory->tail) {
cmp = sl->factory->compare_func(data, previous->links[i]->data);
if (cmp < 0) {
break;
@ -437,7 +435,7 @@ int uniq_skiplist_delete(UniqSkiplist *sl, void *data)
deleted = previous->links[level_index];
for (i=level_index; i>=0; i--) {
while (previous->links[i] != sl->tail &&
while (previous->links[i] != sl->factory->tail &&
previous->links[i] != deleted)
{
previous = previous->links[i];
@ -474,8 +472,8 @@ int uniq_skiplist_find_all(UniqSkiplist *sl, void *data,
previous = uniq_skiplist_get_equal_previous(sl, data, &level_index);
if (previous == NULL) {
iterator->tail = sl->tail;
iterator->current = sl->tail;
iterator->tail = sl->factory->tail;
iterator->current = sl->factory->tail;
return ENOENT;
}
@ -488,18 +486,17 @@ int uniq_skiplist_find_range(UniqSkiplist *sl, void *start_data,
void *end_data, UniqSkiplistIterator *iterator)
{
if (sl->factory->compare_func(start_data, end_data) > 0) {
iterator->current = sl->tail;
iterator->tail = sl->tail;
iterator->current = sl->factory->tail;
iterator->tail = sl->factory->tail;
return EINVAL;
}
iterator->current = uniq_skiplist_get_first_larger_or_equal(sl, start_data);
if (iterator->current == sl->tail) {
iterator->tail = sl->tail;
if (iterator->current == sl->factory->tail) {
iterator->tail = sl->factory->tail;
return ENOENT;
}
iterator->tail = uniq_skiplist_get_first_larger(sl, end_data);
return iterator->current != iterator->tail ? 0 : ENOENT;
}

7
src/uniq_skiplist.h
View File

@ -33,6 +33,7 @@ typedef struct uniq_skiplist_factory
int delay_free_seconds;
skiplist_compare_func compare_func;
uniq_skiplist_free_func free_func;
UniqSkiplistNode *tail; //the tail node for interator
struct fast_mblock_man skiplist_allocator;
struct fast_mblock_man *node_allocators;
} UniqSkiplistFactory;
@ -43,7 +44,6 @@ typedef struct uniq_skiplist
int top_level_index;
int element_count;
UniqSkiplistNode *top; //the top node
UniqSkiplistNode *tail; //the tail node for interator
} UniqSkiplist;
typedef struct uniq_skiplist_iterator {
@ -85,7 +85,7 @@ int uniq_skiplist_find_range(UniqSkiplist *sl, void *start_data,
static inline void uniq_skiplist_iterator(UniqSkiplist *sl, UniqSkiplistIterator *iterator)
{
iterator->current = sl->top->links[0];
iterator->tail = sl->tail;
iterator->tail = sl->factory->tail;
}
static inline void *uniq_skiplist_next(UniqSkiplistIterator *iterator)
@ -103,7 +103,7 @@ static inline void *uniq_skiplist_next(UniqSkiplistIterator *iterator)
static inline bool uniq_skiplist_empty(UniqSkiplist *sl)
{
return sl->top->links[0] == sl->tail;
return sl->top->links[0] == sl->factory->tail;
}
#ifdef __cplusplus
@ -111,4 +111,3 @@ static inline bool uniq_skiplist_empty(UniqSkiplist *sl)
#endif
#endif