pdxx
/
fastdfs
mirror of https://github.com/happyfish100/fastdfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support alignment size for trunk space allocation

pull/484/head
YuQing 2019-12-20 12:02:48 +08:00
parent 13ba0963a3
commit f55d8fafc8
9 changed files with 106 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY
View File

@ -9,6 +9,7 @@ Version 6.05 2019-12-20
* trunk binlog compression support transaction
* support backup binlog file when truncate trunk binlog,
the config item in tracker.conf: trunk_binlog_max_backups
* support alignment size for trunk space allocation
Version 6.04 2019-12-05
* storage_report_ip_changed ignore result EEXIST

9
conf/tracker.conf
View File

@ -152,7 +152,14 @@ slot_min_size = 256
# store the upload file to trunk file when it's size <= this value
# default value is 16MB
# since V3.00
slot_max_size = 4MB
slot_max_size = 1MB
# the alignment size to allocate the trunk space
# default value is 0 (never align)
# since V6.05
# NOTE: the larger the alignment size, the less likely of disk
# fragmentation, but the more space is wasted.
trunk_alloc_alignment_size = 512
# the trunk file size, should >= 4MB
# default value is 64MB

14
storage/storage_param_getter.c
View File

@ -114,7 +114,13 @@ int storage_get_params_from_tracker()
g_trunk_file_size = iniGetIntValue(NULL, "trunk_file_size",
&iniContext, 64 * 1024 * 1024);
g_slot_max_size = iniGetIntValue(NULL, "slot_max_size",
&iniContext, g_trunk_file_size / 2);
&iniContext, g_trunk_file_size / 4);
g_trunk_alloc_alignment_size = iniGetIntValue(NULL,
"trunk_alloc_alignment_size", &iniContext, 0);
if (g_slot_min_size < g_trunk_alloc_alignment_size)
{
g_slot_min_size = g_trunk_alloc_alignment_size;
}
g_trunk_create_file_advance = iniGetBoolValue(NULL,
"trunk_create_file_advance", &iniContext, false);
@ -188,7 +194,8 @@ int storage_get_params_from_tracker()
"reserved_storage_space=%s, "
"use_trunk_file=%d, "
"slot_min_size=%d, "
"slot_max_size=%d MB, "
"slot_max_size=%d KB, "
"trunk_alloc_alignment_size=%d, "
"trunk_file_size=%d MB, "
"trunk_create_file_advance=%d, "
"trunk_create_file_time_base=%02d:%02d, "
@ -207,7 +214,8 @@ int storage_get_params_from_tracker()
g_store_path_mode, fdfs_storage_reserved_space_to_string(
&g_storage_reserved_space, reserved_space_str),
g_if_use_trunk_file, g_slot_min_size,
g_slot_max_size / FDFS_ONE_MB,
g_slot_max_size / 1024,
g_trunk_alloc_alignment_size,
g_trunk_file_size / FDFS_ONE_MB,
g_trunk_create_file_advance,
g_trunk_create_file_time_base.hour,

47
storage/trunk_mgr/trunk_mem.c
View File

@ -42,9 +42,10 @@
#define STORAGE_TRUNK_INIT_FLAG_DESTROYING 1
#define STORAGE_TRUNK_INIT_FLAG_DONE 2
int g_slot_min_size;
int g_trunk_file_size;
int g_slot_max_size;
int g_slot_min_size = 0;
int g_slot_max_size = 0;
int g_trunk_alloc_alignment_size = 0;
int g_trunk_file_size = 0;
int g_store_path_mode = FDFS_STORE_PATH_ROUND_ROBIN;
FDFSStorageReservedSpace g_storage_reserved_space = {
TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB};
@ -1748,10 +1749,33 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
FDFSTrunkNode *pPreviousNode;
FDFSTrunkNode *pTrunkNode;
int result;
int aligned_size;
int remain;
STORAGE_TRUNK_CHECK_STATUS();
target_slot.size = (size > g_slot_min_size) ? size : g_slot_min_size;
if (size <= g_slot_min_size)
{
aligned_size = g_slot_min_size;
}
else if (g_trunk_alloc_alignment_size == 0)
{
aligned_size = size;
}
else
{
remain = size % g_trunk_alloc_alignment_size;
if (remain == 0)
{
aligned_size = size;
}
else
{
aligned_size = size + (g_trunk_alloc_alignment_size - remain);
}
}
target_slot.size = aligned_size;
target_slot.head = NULL;
pPreviousNode = NULL;
@ -1759,7 +1783,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
pthread_mutex_lock(&trunk_mem_lock);
while (1)
{
pSlot = (FDFSTrunkSlot *)avl_tree_find_ge(tree_info_by_sizes \
pSlot = (FDFSTrunkSlot *)avl_tree_find_ge(tree_info_by_sizes
+ pResult->path.store_path_index, &target_slot);
if (pSlot == NULL)
{
@ -1768,7 +1792,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
pPreviousNode = NULL;
pTrunkNode = pSlot->head;
while (pTrunkNode != NULL && \
while (pTrunkNode != NULL &&
pTrunkNode->trunk.status == FDFS_TRUNK_STATUS_HOLD)
{
pPreviousNode = pTrunkNode;
@ -1790,7 +1814,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
pSlot->head = pTrunkNode->next;
if (pSlot->head == NULL)
{
trunk_delete_size_tree_entry(pResult->path. \
trunk_delete_size_tree_entry(pResult->path.
store_path_index, pSlot);
}
}
@ -1803,7 +1827,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
}
else
{
pTrunkNode = trunk_create_trunk_file(pResult->path. \
pTrunkNode = trunk_create_trunk_file(pResult->path.
store_path_index, &result);
if (pTrunkNode == NULL)
{
@ -1813,7 +1837,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
}
pthread_mutex_unlock(&trunk_mem_lock);
result = trunk_split(pTrunkNode, size);
result = trunk_split(pTrunkNode, aligned_size);
if (result != 0)
{
return result;
@ -1823,10 +1847,13 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
result = trunk_add_free_block(pTrunkNode, true);
if (result == 0)
{
memcpy(pResult, &(pTrunkNode->trunk), \
memcpy(pResult, &(pTrunkNode->trunk),
sizeof(FDFSTrunkFullInfo));
}
logInfo("alloc size: %d, aligned_size: %d, alloced trunk size: %d",
size, aligned_size, pResult->file.size);
return result;
}

1
storage/trunk_mgr/trunk_mem.h
View File

@ -39,6 +39,7 @@ extern "C" {
extern int g_slot_min_size; //slot min size, such as 256 bytes
extern int g_slot_max_size; //slot max size
extern int g_trunk_alloc_alignment_size; //the alignment size for trunk alloc
extern int g_trunk_file_size; //the trunk file size, such as 64MB
extern int g_store_path_mode; //store which path mode, fetch from tracker
extern FDFSStorageReservedSpace g_storage_reserved_space; //fetch from tracker

74
tracker/tracker_func.c
View File

@ -504,19 +504,19 @@ int tracker_load_from_conf_file(const char *filename, \
g_trunk_file_size = (int)trunk_file_size;
if (g_trunk_file_size < 4 * 1024 * 1024)
{
logWarning("file: "__FILE__", line: %d, " \
"item \"trunk_file_size\" %d is too small, " \
logWarning("file: "__FILE__", line: %d, "
"item \"trunk_file_size\" %d is too small, "
"change to 4MB", __LINE__, g_trunk_file_size);
g_trunk_file_size = 4 * 1024 * 1024;
}
pSlotMaxSize = iniGetStrValue(NULL, \
pSlotMaxSize = iniGetStrValue(NULL,
"slot_max_size", &iniContext);
if (pSlotMaxSize == NULL)
{
slot_max_size = g_trunk_file_size / 2;
slot_max_size = g_trunk_file_size / 8;
}
else if ((result=parse_bytes(pSlotMaxSize, 1, \
else if ((result=parse_bytes(pSlotMaxSize, 1,
&slot_max_size)) != 0)
{
return result;
@ -524,26 +524,26 @@ int tracker_load_from_conf_file(const char *filename, \
g_slot_max_size = (int)slot_max_size;
if (g_slot_max_size <= g_slot_min_size)
{
logError("file: "__FILE__", line: %d, " \
"item \"slot_max_size\" %d is invalid, " \
"which <= slot_min_size: %d", \
logError("file: "__FILE__", line: %d, "
"item \"slot_max_size\" %d is invalid, "
"which <= slot_min_size: %d",
__LINE__, g_slot_max_size, g_slot_min_size);
result = EINVAL;
break;
}
if (g_slot_max_size > g_trunk_file_size / 2)
{
logWarning("file: "__FILE__", line: %d, " \
"item \"slot_max_size\": %d is too large, " \
"change to %d", __LINE__, g_slot_max_size, \
logWarning("file: "__FILE__", line: %d, "
"item \"slot_max_size\": %d is too large, "
"change to %d", __LINE__, g_slot_max_size,
g_trunk_file_size / 2);
g_slot_max_size = g_trunk_file_size / 2;
}
g_trunk_create_file_advance = iniGetBoolValue(NULL, \
g_trunk_create_file_advance = iniGetBoolValue(NULL,
"trunk_create_file_advance", &iniContext, false);
if ((result=get_time_item_from_conf(&iniContext, \
"trunk_create_file_time_base", \
if ((result=get_time_item_from_conf(&iniContext,
"trunk_create_file_time_base",
&g_trunk_create_file_time_base, 2, 0)) != 0)
{
return result;
@ -579,13 +579,25 @@ int tracker_load_from_conf_file(const char *filename, \
g_trunk_binlog_max_backups = iniGetIntValue(NULL,
"trunk_binlog_max_backups", &iniContext, 0);
g_trunk_init_check_occupying = iniGetBoolValue(NULL, \
g_trunk_alloc_alignment_size = iniGetIntValue(NULL,
"trunk_alloc_alignment_size", &iniContext, 0);
if (g_slot_min_size < g_trunk_alloc_alignment_size)
{
logWarning("file: "__FILE__", line: %d, "
"item \"slot_min_size\": %d < "
"\"trunk_alloc_alignment_size\": %d, "
"change to %d", __LINE__, g_slot_min_size,
g_trunk_alloc_alignment_size);
g_slot_min_size = g_trunk_alloc_alignment_size;
}
g_trunk_init_check_occupying = iniGetBoolValue(NULL,
"trunk_init_check_occupying", &iniContext, false);
g_trunk_init_reload_from_binlog = iniGetBoolValue(NULL, \
g_trunk_init_reload_from_binlog = iniGetBoolValue(NULL,
"trunk_init_reload_from_binlog", &iniContext, false);
if ((result=tracker_load_storage_id_info( \
if ((result=tracker_load_storage_id_info(
filename, &iniContext)) != 0)
{
return result;
@ -604,40 +616,40 @@ int tracker_load_from_conf_file(const char *filename, \
log_set_compress_log_days_before(g_compress_error_log_days_before);
}
if ((result=get_time_item_from_conf(&iniContext, \
"error_log_rotate_time", &g_error_log_rotate_time, \
if ((result=get_time_item_from_conf(&iniContext,
"error_log_rotate_time", &g_error_log_rotate_time,
0, 0)) != 0)
{
break;
}
pRotateErrorLogSize = iniGetStrValue(NULL, \
pRotateErrorLogSize = iniGetStrValue(NULL,
"rotate_error_log_size", &iniContext);
if (pRotateErrorLogSize == NULL)
{
rotate_error_log_size = 0;
}
else if ((result=parse_bytes(pRotateErrorLogSize, 1, \
else if ((result=parse_bytes(pRotateErrorLogSize, 1,
&rotate_error_log_size)) != 0)
{
break;
}
if (rotate_error_log_size > 0 && \
if (rotate_error_log_size > 0 &&
rotate_error_log_size < FDFS_ONE_MB)
{
logWarning("file: "__FILE__", line: %d, " \
"item \"rotate_error_log_size\": " \
"%"PRId64" is too small, " \
"change to 1 MB", __LINE__, \
logWarning("file: "__FILE__", line: %d, "
"item \"rotate_error_log_size\": "
"%"PRId64" is too small, "
"change to 1 MB", __LINE__,
rotate_error_log_size);
rotate_error_log_size = FDFS_ONE_MB;
}
fdfs_set_log_rotate_size(&g_log_context, rotate_error_log_size);
g_log_file_keep_days = iniGetIntValue(NULL, \
g_log_file_keep_days = iniGetIntValue(NULL,
"log_file_keep_days", &iniContext, 0);
g_store_slave_file_use_link = iniGetBoolValue(NULL, \
g_store_slave_file_use_link = iniGetBoolValue(NULL,
"store_slave_file_use_link", &iniContext, false);
if ((result=fdfs_connection_pool_init(filename, &iniContext)) != 0)
@ -753,7 +765,8 @@ int tracker_load_from_conf_file(const char *filename, \
"storage_sync_file_max_time=%ds, "
"use_trunk_file=%d, "
"slot_min_size=%d, "
"slot_max_size=%d MB, "
"slot_max_size=%d KB, "
"trunk_alloc_alignment_size=%d, "
"trunk_file_size=%d MB, "
"trunk_create_file_advance=%d, "
"trunk_create_file_time_base=%02d:%02d, "
@ -794,7 +807,8 @@ int tracker_load_from_conf_file(const char *filename, \
g_storage_sync_file_max_delay,
g_storage_sync_file_max_time,
g_if_use_trunk_file, g_slot_min_size,
g_slot_max_size / FDFS_ONE_MB,
g_slot_max_size / 1024,
g_trunk_alloc_alignment_size,
g_trunk_file_size / FDFS_ONE_MB,
g_trunk_create_file_advance,
g_trunk_create_file_time_base.hour,

1
tracker/tracker_global.c
View File

@ -60,6 +60,7 @@ int g_trunk_create_file_interval = 86400;
int g_trunk_compress_binlog_interval = 0;
int g_trunk_compress_binlog_min_interval = 0;
int g_trunk_binlog_max_backups = 0;
int g_trunk_alloc_alignment_size = 0;
int64_t g_trunk_create_file_space_threshold = 0;
time_t g_up_time = 0;

1
tracker/tracker_global.h
View File

@ -84,6 +84,7 @@ extern int g_trunk_create_file_interval;
extern int g_trunk_compress_binlog_interval;
extern int g_trunk_compress_binlog_min_interval;
extern int g_trunk_binlog_max_backups;
extern int g_trunk_alloc_alignment_size;
extern int64_t g_trunk_create_file_space_threshold;
extern time_t g_up_time;

2
tracker/tracker_service.c
View File

@ -687,6 +687,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
"use_trunk_file=%d\n"
"slot_min_size=%d\n"
"slot_max_size=%d\n"
"trunk_alloc_alignment_size=%d\n"
"trunk_file_size=%d\n"
"trunk_create_file_advance=%d\n"
"trunk_create_file_time_base=%02d:%02d\n"
@ -707,6 +708,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
&g_storage_reserved_space, reserved_space_str),
g_if_use_trunk_file,
g_slot_min_size, g_slot_max_size,
g_trunk_alloc_alignment_size,
g_trunk_file_size, g_trunk_create_file_advance,
g_trunk_create_file_time_base.hour,
g_trunk_create_file_time_base.minute,