pdxx
/
fastdfs
mirror of https://github.com/happyfish100/fastdfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support alignment size for trunk space allocation

pull/484/head
YuQing 2019-12-20 12:02:48 +08:00
parent 13ba0963a3
commit f55d8fafc8
9 changed files with 106 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY
View File

@ -9,6 +9,7 @@ Version 6.05 2019-12-20
* trunk binlog compression support transaction * trunk binlog compression support transaction
* support backup binlog file when truncate trunk binlog, * support backup binlog file when truncate trunk binlog,
the config item in tracker.conf: trunk_binlog_max_backups the config item in tracker.conf: trunk_binlog_max_backups
* support alignment size for trunk space allocation
Version 6.04 2019-12-05 Version 6.04 2019-12-05
* storage_report_ip_changed ignore result EEXIST * storage_report_ip_changed ignore result EEXIST

9
conf/tracker.conf
View File

@ -152,7 +152,14 @@ slot_min_size = 256
# store the upload file to trunk file when it's size <= this value # store the upload file to trunk file when it's size <= this value
# default value is 16MB # default value is 16MB
# since V3.00 # since V3.00
slot_max_size = 4MB slot_max_size = 1MB
# the alignment size to allocate the trunk space
# default value is 0 (never align)
# since V6.05
# NOTE: the larger the alignment size, the less likely of disk
# fragmentation, but the more space is wasted.
trunk_alloc_alignment_size = 512
# the trunk file size, should >= 4MB # the trunk file size, should >= 4MB
# default value is 64MB # default value is 64MB

14
storage/storage_param_getter.c
View File

@ -114,7 +114,13 @@ int storage_get_params_from_tracker()
g_trunk_file_size = iniGetIntValue(NULL, "trunk_file_size", g_trunk_file_size = iniGetIntValue(NULL, "trunk_file_size",
&iniContext, 64 * 1024 * 1024); &iniContext, 64 * 1024 * 1024);
g_slot_max_size = iniGetIntValue(NULL, "slot_max_size", g_slot_max_size = iniGetIntValue(NULL, "slot_max_size",
&iniContext, g_trunk_file_size / 2); &iniContext, g_trunk_file_size / 4);
g_trunk_alloc_alignment_size = iniGetIntValue(NULL,
"trunk_alloc_alignment_size", &iniContext, 0);
if (g_slot_min_size < g_trunk_alloc_alignment_size)
{
g_slot_min_size = g_trunk_alloc_alignment_size;
}
g_trunk_create_file_advance = iniGetBoolValue(NULL, g_trunk_create_file_advance = iniGetBoolValue(NULL,
"trunk_create_file_advance", &iniContext, false); "trunk_create_file_advance", &iniContext, false);
@ -188,7 +194,8 @@ int storage_get_params_from_tracker()
"reserved_storage_space=%s, " "reserved_storage_space=%s, "
"use_trunk_file=%d, " "use_trunk_file=%d, "
"slot_min_size=%d, " "slot_min_size=%d, "
"slot_max_size=%d MB, " "slot_max_size=%d KB, "
"trunk_alloc_alignment_size=%d, "
"trunk_file_size=%d MB, " "trunk_file_size=%d MB, "
"trunk_create_file_advance=%d, " "trunk_create_file_advance=%d, "
"trunk_create_file_time_base=%02d:%02d, " "trunk_create_file_time_base=%02d:%02d, "
@ -207,7 +214,8 @@ int storage_get_params_from_tracker()
g_store_path_mode, fdfs_storage_reserved_space_to_string( g_store_path_mode, fdfs_storage_reserved_space_to_string(
&g_storage_reserved_space, reserved_space_str), &g_storage_reserved_space, reserved_space_str),
g_if_use_trunk_file, g_slot_min_size, g_if_use_trunk_file, g_slot_min_size,
g_slot_max_size / FDFS_ONE_MB, g_slot_max_size / 1024,
g_trunk_alloc_alignment_size,
g_trunk_file_size / FDFS_ONE_MB, g_trunk_file_size / FDFS_ONE_MB,
g_trunk_create_file_advance, g_trunk_create_file_advance,
g_trunk_create_file_time_base.hour, g_trunk_create_file_time_base.hour,

47
storage/trunk_mgr/trunk_mem.c
View File

@ -42,9 +42,10 @@
#define STORAGE_TRUNK_INIT_FLAG_DESTROYING 1 #define STORAGE_TRUNK_INIT_FLAG_DESTROYING 1
#define STORAGE_TRUNK_INIT_FLAG_DONE 2 #define STORAGE_TRUNK_INIT_FLAG_DONE 2
int g_slot_min_size; int g_slot_min_size = 0;
int g_trunk_file_size; int g_slot_max_size = 0;
int g_slot_max_size; int g_trunk_alloc_alignment_size = 0;
int g_trunk_file_size = 0;
int g_store_path_mode = FDFS_STORE_PATH_ROUND_ROBIN; int g_store_path_mode = FDFS_STORE_PATH_ROUND_ROBIN;
FDFSStorageReservedSpace g_storage_reserved_space = { FDFSStorageReservedSpace g_storage_reserved_space = {
TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB}; TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB};
@ -1748,10 +1749,33 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
FDFSTrunkNode *pPreviousNode; FDFSTrunkNode *pPreviousNode;
FDFSTrunkNode *pTrunkNode; FDFSTrunkNode *pTrunkNode;
int result; int result;
int aligned_size;
int remain;
STORAGE_TRUNK_CHECK_STATUS(); STORAGE_TRUNK_CHECK_STATUS();
target_slot.size = (size > g_slot_min_size) ? size : g_slot_min_size; if (size <= g_slot_min_size)
{
aligned_size = g_slot_min_size;
}
else if (g_trunk_alloc_alignment_size == 0)
{
aligned_size = size;
}
else
{
remain = size % g_trunk_alloc_alignment_size;
if (remain == 0)
{
aligned_size = size;
}
else
{
aligned_size = size + (g_trunk_alloc_alignment_size - remain);
}
}
target_slot.size = aligned_size;
target_slot.head = NULL; target_slot.head = NULL;
pPreviousNode = NULL; pPreviousNode = NULL;
@ -1759,7 +1783,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
pthread_mutex_lock(&trunk_mem_lock); pthread_mutex_lock(&trunk_mem_lock);
while (1) while (1)
{ {
pSlot = (FDFSTrunkSlot *)avl_tree_find_ge(tree_info_by_sizes \ pSlot = (FDFSTrunkSlot *)avl_tree_find_ge(tree_info_by_sizes
+ pResult->path.store_path_index, &target_slot); + pResult->path.store_path_index, &target_slot);
if (pSlot == NULL) if (pSlot == NULL)
{ {
@ -1768,7 +1792,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
pPreviousNode = NULL; pPreviousNode = NULL;
pTrunkNode = pSlot->head; pTrunkNode = pSlot->head;
while (pTrunkNode != NULL && \ while (pTrunkNode != NULL &&
pTrunkNode->trunk.status == FDFS_TRUNK_STATUS_HOLD) pTrunkNode->trunk.status == FDFS_TRUNK_STATUS_HOLD)
{ {
pPreviousNode = pTrunkNode; pPreviousNode = pTrunkNode;
@ -1790,7 +1814,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
pSlot->head = pTrunkNode->next; pSlot->head = pTrunkNode->next;
if (pSlot->head == NULL) if (pSlot->head == NULL)
{ {
trunk_delete_size_tree_entry(pResult->path. \ trunk_delete_size_tree_entry(pResult->path.
store_path_index, pSlot); store_path_index, pSlot);
} }
} }
@ -1803,7 +1827,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
} }
else else
{ {
pTrunkNode = trunk_create_trunk_file(pResult->path. \ pTrunkNode = trunk_create_trunk_file(pResult->path.
store_path_index, &result); store_path_index, &result);
if (pTrunkNode == NULL) if (pTrunkNode == NULL)
{ {
@ -1813,7 +1837,7 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
} }
pthread_mutex_unlock(&trunk_mem_lock); pthread_mutex_unlock(&trunk_mem_lock);
result = trunk_split(pTrunkNode, size); result = trunk_split(pTrunkNode, aligned_size);
if (result != 0) if (result != 0)
{ {
return result; return result;
@ -1823,10 +1847,13 @@ int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult)
result = trunk_add_free_block(pTrunkNode, true); result = trunk_add_free_block(pTrunkNode, true);
if (result == 0) if (result == 0)
{ {
memcpy(pResult, &(pTrunkNode->trunk), \ memcpy(pResult, &(pTrunkNode->trunk),
sizeof(FDFSTrunkFullInfo)); sizeof(FDFSTrunkFullInfo));
} }
logInfo("alloc size: %d, aligned_size: %d, alloced trunk size: %d",
size, aligned_size, pResult->file.size);
return result; return result;
} }

1
storage/trunk_mgr/trunk_mem.h
View File

@ -39,6 +39,7 @@ extern "C" {
extern int g_slot_min_size; //slot min size, such as 256 bytes extern int g_slot_min_size; //slot min size, such as 256 bytes
extern int g_slot_max_size; //slot max size extern int g_slot_max_size; //slot max size
extern int g_trunk_alloc_alignment_size; //the alignment size for trunk alloc
extern int g_trunk_file_size; //the trunk file size, such as 64MB extern int g_trunk_file_size; //the trunk file size, such as 64MB
extern int g_store_path_mode; //store which path mode, fetch from tracker extern int g_store_path_mode; //store which path mode, fetch from tracker
extern FDFSStorageReservedSpace g_storage_reserved_space; //fetch from tracker extern FDFSStorageReservedSpace g_storage_reserved_space; //fetch from tracker

74
tracker/tracker_func.c
View File

@ -504,19 +504,19 @@ int tracker_load_from_conf_file(const char *filename, \
g_trunk_file_size = (int)trunk_file_size; g_trunk_file_size = (int)trunk_file_size;
if (g_trunk_file_size < 4 * 1024 * 1024) if (g_trunk_file_size < 4 * 1024 * 1024)
{ {
logWarning("file: "__FILE__", line: %d, " \ logWarning("file: "__FILE__", line: %d, "
"item \"trunk_file_size\" %d is too small, " \ "item \"trunk_file_size\" %d is too small, "
"change to 4MB", __LINE__, g_trunk_file_size); "change to 4MB", __LINE__, g_trunk_file_size);
g_trunk_file_size = 4 * 1024 * 1024; g_trunk_file_size = 4 * 1024 * 1024;
} }
pSlotMaxSize = iniGetStrValue(NULL, \ pSlotMaxSize = iniGetStrValue(NULL,
"slot_max_size", &iniContext); "slot_max_size", &iniContext);
if (pSlotMaxSize == NULL) if (pSlotMaxSize == NULL)
{ {
slot_max_size = g_trunk_file_size / 2; slot_max_size = g_trunk_file_size / 8;
} }
else if ((result=parse_bytes(pSlotMaxSize, 1, \ else if ((result=parse_bytes(pSlotMaxSize, 1,
&slot_max_size)) != 0) &slot_max_size)) != 0)
{ {
return result; return result;
@ -524,26 +524,26 @@ int tracker_load_from_conf_file(const char *filename, \
g_slot_max_size = (int)slot_max_size; g_slot_max_size = (int)slot_max_size;
if (g_slot_max_size <= g_slot_min_size) if (g_slot_max_size <= g_slot_min_size)
{ {
logError("file: "__FILE__", line: %d, " \ logError("file: "__FILE__", line: %d, "
"item \"slot_max_size\" %d is invalid, " \ "item \"slot_max_size\" %d is invalid, "
"which <= slot_min_size: %d", \ "which <= slot_min_size: %d",
__LINE__, g_slot_max_size, g_slot_min_size); __LINE__, g_slot_max_size, g_slot_min_size);
result = EINVAL; result = EINVAL;
break; break;
} }
if (g_slot_max_size > g_trunk_file_size / 2) if (g_slot_max_size > g_trunk_file_size / 2)
{ {
logWarning("file: "__FILE__", line: %d, " \ logWarning("file: "__FILE__", line: %d, "
"item \"slot_max_size\": %d is too large, " \ "item \"slot_max_size\": %d is too large, "
"change to %d", __LINE__, g_slot_max_size, \ "change to %d", __LINE__, g_slot_max_size,
g_trunk_file_size / 2); g_trunk_file_size / 2);
g_slot_max_size = g_trunk_file_size / 2; g_slot_max_size = g_trunk_file_size / 2;
} }
g_trunk_create_file_advance = iniGetBoolValue(NULL, \ g_trunk_create_file_advance = iniGetBoolValue(NULL,
"trunk_create_file_advance", &iniContext, false); "trunk_create_file_advance", &iniContext, false);
if ((result=get_time_item_from_conf(&iniContext, \ if ((result=get_time_item_from_conf(&iniContext,
"trunk_create_file_time_base", \ "trunk_create_file_time_base",
&g_trunk_create_file_time_base, 2, 0)) != 0) &g_trunk_create_file_time_base, 2, 0)) != 0)
{ {
return result; return result;
@ -579,13 +579,25 @@ int tracker_load_from_conf_file(const char *filename, \
g_trunk_binlog_max_backups = iniGetIntValue(NULL, g_trunk_binlog_max_backups = iniGetIntValue(NULL,
"trunk_binlog_max_backups", &iniContext, 0); "trunk_binlog_max_backups", &iniContext, 0);
g_trunk_init_check_occupying = iniGetBoolValue(NULL, \ g_trunk_alloc_alignment_size = iniGetIntValue(NULL,
"trunk_alloc_alignment_size", &iniContext, 0);
if (g_slot_min_size < g_trunk_alloc_alignment_size)
{
logWarning("file: "__FILE__", line: %d, "
"item \"slot_min_size\": %d < "
"\"trunk_alloc_alignment_size\": %d, "
"change to %d", __LINE__, g_slot_min_size,
g_trunk_alloc_alignment_size);
g_slot_min_size = g_trunk_alloc_alignment_size;
}
g_trunk_init_check_occupying = iniGetBoolValue(NULL,
"trunk_init_check_occupying", &iniContext, false); "trunk_init_check_occupying", &iniContext, false);
g_trunk_init_reload_from_binlog = iniGetBoolValue(NULL, \ g_trunk_init_reload_from_binlog = iniGetBoolValue(NULL,
"trunk_init_reload_from_binlog", &iniContext, false); "trunk_init_reload_from_binlog", &iniContext, false);
if ((result=tracker_load_storage_id_info( \ if ((result=tracker_load_storage_id_info(
filename, &iniContext)) != 0) filename, &iniContext)) != 0)
{ {
return result; return result;
@ -604,40 +616,40 @@ int tracker_load_from_conf_file(const char *filename, \
log_set_compress_log_days_before(g_compress_error_log_days_before); log_set_compress_log_days_before(g_compress_error_log_days_before);
} }
if ((result=get_time_item_from_conf(&iniContext, \ if ((result=get_time_item_from_conf(&iniContext,
"error_log_rotate_time", &g_error_log_rotate_time, \ "error_log_rotate_time", &g_error_log_rotate_time,
0, 0)) != 0) 0, 0)) != 0)
{ {
break; break;
} }
pRotateErrorLogSize = iniGetStrValue(NULL, \ pRotateErrorLogSize = iniGetStrValue(NULL,
"rotate_error_log_size", &iniContext); "rotate_error_log_size", &iniContext);
if (pRotateErrorLogSize == NULL) if (pRotateErrorLogSize == NULL)
{ {
rotate_error_log_size = 0; rotate_error_log_size = 0;
} }
else if ((result=parse_bytes(pRotateErrorLogSize, 1, \ else if ((result=parse_bytes(pRotateErrorLogSize, 1,
&rotate_error_log_size)) != 0) &rotate_error_log_size)) != 0)
{ {
break; break;
} }
if (rotate_error_log_size > 0 && \ if (rotate_error_log_size > 0 &&
rotate_error_log_size < FDFS_ONE_MB) rotate_error_log_size < FDFS_ONE_MB)
{ {
logWarning("file: "__FILE__", line: %d, " \ logWarning("file: "__FILE__", line: %d, "
"item \"rotate_error_log_size\": " \ "item \"rotate_error_log_size\": "
"%"PRId64" is too small, " \ "%"PRId64" is too small, "
"change to 1 MB", __LINE__, \ "change to 1 MB", __LINE__,
rotate_error_log_size); rotate_error_log_size);
rotate_error_log_size = FDFS_ONE_MB; rotate_error_log_size = FDFS_ONE_MB;
} }
fdfs_set_log_rotate_size(&g_log_context, rotate_error_log_size); fdfs_set_log_rotate_size(&g_log_context, rotate_error_log_size);
g_log_file_keep_days = iniGetIntValue(NULL, \ g_log_file_keep_days = iniGetIntValue(NULL,
"log_file_keep_days", &iniContext, 0); "log_file_keep_days", &iniContext, 0);
g_store_slave_file_use_link = iniGetBoolValue(NULL, \ g_store_slave_file_use_link = iniGetBoolValue(NULL,
"store_slave_file_use_link", &iniContext, false); "store_slave_file_use_link", &iniContext, false);
if ((result=fdfs_connection_pool_init(filename, &iniContext)) != 0) if ((result=fdfs_connection_pool_init(filename, &iniContext)) != 0)
@ -753,7 +765,8 @@ int tracker_load_from_conf_file(const char *filename, \
"storage_sync_file_max_time=%ds, " "storage_sync_file_max_time=%ds, "
"use_trunk_file=%d, " "use_trunk_file=%d, "
"slot_min_size=%d, " "slot_min_size=%d, "
"slot_max_size=%d MB, " "slot_max_size=%d KB, "
"trunk_alloc_alignment_size=%d, "
"trunk_file_size=%d MB, " "trunk_file_size=%d MB, "
"trunk_create_file_advance=%d, " "trunk_create_file_advance=%d, "
"trunk_create_file_time_base=%02d:%02d, " "trunk_create_file_time_base=%02d:%02d, "
@ -794,7 +807,8 @@ int tracker_load_from_conf_file(const char *filename, \
g_storage_sync_file_max_delay, g_storage_sync_file_max_delay,
g_storage_sync_file_max_time, g_storage_sync_file_max_time,
g_if_use_trunk_file, g_slot_min_size, g_if_use_trunk_file, g_slot_min_size,
g_slot_max_size / FDFS_ONE_MB, g_slot_max_size / 1024,
g_trunk_alloc_alignment_size,
g_trunk_file_size / FDFS_ONE_MB, g_trunk_file_size / FDFS_ONE_MB,
g_trunk_create_file_advance, g_trunk_create_file_advance,
g_trunk_create_file_time_base.hour, g_trunk_create_file_time_base.hour,

1
tracker/tracker_global.c
View File

@ -60,6 +60,7 @@ int g_trunk_create_file_interval = 86400;
int g_trunk_compress_binlog_interval = 0; int g_trunk_compress_binlog_interval = 0;
int g_trunk_compress_binlog_min_interval = 0; int g_trunk_compress_binlog_min_interval = 0;
int g_trunk_binlog_max_backups = 0; int g_trunk_binlog_max_backups = 0;
int g_trunk_alloc_alignment_size = 0;
int64_t g_trunk_create_file_space_threshold = 0; int64_t g_trunk_create_file_space_threshold = 0;
time_t g_up_time = 0; time_t g_up_time = 0;

1
tracker/tracker_global.h
View File

@ -84,6 +84,7 @@ extern int g_trunk_create_file_interval;
extern int g_trunk_compress_binlog_interval; extern int g_trunk_compress_binlog_interval;
extern int g_trunk_compress_binlog_min_interval; extern int g_trunk_compress_binlog_min_interval;
extern int g_trunk_binlog_max_backups; extern int g_trunk_binlog_max_backups;
extern int g_trunk_alloc_alignment_size;
extern int64_t g_trunk_create_file_space_threshold; extern int64_t g_trunk_create_file_space_threshold;
extern time_t g_up_time; extern time_t g_up_time;

2
tracker/tracker_service.c
View File

@ -687,6 +687,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
"use_trunk_file=%d\n" "use_trunk_file=%d\n"
"slot_min_size=%d\n" "slot_min_size=%d\n"
"slot_max_size=%d\n" "slot_max_size=%d\n"
"trunk_alloc_alignment_size=%d\n"
"trunk_file_size=%d\n" "trunk_file_size=%d\n"
"trunk_create_file_advance=%d\n" "trunk_create_file_advance=%d\n"
"trunk_create_file_time_base=%02d:%02d\n" "trunk_create_file_time_base=%02d:%02d\n"
@ -707,6 +708,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
&g_storage_reserved_space, reserved_space_str), &g_storage_reserved_space, reserved_space_str),
g_if_use_trunk_file, g_if_use_trunk_file,
g_slot_min_size, g_slot_max_size, g_slot_min_size, g_slot_max_size,
g_trunk_alloc_alignment_size,
g_trunk_file_size, g_trunk_create_file_advance, g_trunk_file_size, g_trunk_create_file_advance,
g_trunk_create_file_time_base.hour, g_trunk_create_file_time_base.hour,
g_trunk_create_file_time_base.minute, g_trunk_create_file_time_base.minute,