pdxx
/
fastdfs
mirror of https://github.com/happyfish100/fastdfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support delete unused trunk files

pull/484/head
YuQing 2019-12-23 16:11:18 +08:00
parent 49d51e949b
commit b7447e5903
9 changed files with 105 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
HISTORY
View File

@ -6,11 +6,19 @@ Version 6.05 2019-12-23
* trunk server support compress the trunk binlog periodically,
the config items in tracker.conf: trunk_compress_binlog_interval
and trunk_compress_binlog_time_base
* trunk binlog compression support transaction
* support backup binlog file when truncate trunk binlog,
the config item in tracker.conf: trunk_binlog_max_backups
* support alignment size for trunk space allocation
the config item in tracker.conf: trunk_alloc_alignment_size
* support merge free trunk spaces
the config item in tracker.conf: trunk_free_space_merge
* support delete unused trunk files
the config item in tracker.conf: delete_unused_trunk_files
Version 6.04 2019-12-05
* storage_report_ip_changed ignore result EEXIST

7
conf/tracker.conf
View File

@ -146,7 +146,7 @@ use_trunk_file = false
# the min slot size, should <= 4KB
# default value is 256 bytes
# since V3.00
slot_min_size = 256
slot_min_size = 512
# the max slot size, should > slot_min_size
# store the upload file to trunk file when it's size <= this value
@ -166,6 +166,11 @@ trunk_alloc_alignment_size = 512
# since V6.05
trunk_free_space_merge = true
# if delete / reclaim the unused trunk files
# default value is false
# since V6.05
delete_unused_trunk_files = true
# the trunk file size, should >= 4MB
# default value is 64MB
# since V3.00

4
storage/storage_param_getter.c
View File

@ -144,6 +144,8 @@ int storage_get_params_from_tracker()
"trunk_init_reload_from_binlog", &iniContext, false);
g_trunk_free_space_merge = iniGetBoolValue(NULL,
"trunk_free_space_merge", &iniContext, false);
g_delete_unused_trunk_files = iniGetBoolValue(NULL,
"delete_unused_trunk_files", &iniContext, false);
g_trunk_compress_binlog_min_interval = iniGetIntValue(NULL,
"trunk_compress_binlog_min_interval", &iniContext, 0);
g_trunk_compress_binlog_interval = iniGetIntValue(NULL,
@ -206,6 +208,7 @@ int storage_get_params_from_tracker()
"trunk_init_check_occupying=%d, "
"trunk_init_reload_from_binlog=%d, "
"trunk_free_space_merge=%d, "
"delete_unused_trunk_files=%d, "
"trunk_compress_binlog_min_interval=%d, "
"trunk_compress_binlog_interval=%d, "
"trunk_compress_binlog_time_base=%02d:%02d, "
@ -228,6 +231,7 @@ int storage_get_params_from_tracker()
(FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying,
g_trunk_init_reload_from_binlog,
g_trunk_free_space_merge,
g_delete_unused_trunk_files,
g_trunk_compress_binlog_min_interval,
g_trunk_compress_binlog_interval,
g_trunk_compress_binlog_time_base.hour,

81
storage/trunk_mgr/trunk_mem.c
View File

@ -65,6 +65,7 @@ bool g_trunk_create_file_advance = false;
bool g_trunk_init_check_occupying = false;
bool g_trunk_init_reload_from_binlog = false;
bool g_trunk_free_space_merge = false;
bool g_delete_unused_trunk_files = false;
int g_trunk_binlog_compress_stage = STORAGE_TRUNK_COMPRESS_STAGE_NONE;
volatile int64_t g_trunk_total_free_space = 0;
int64_t g_trunk_create_file_space_threshold = 0;
@ -542,17 +543,20 @@ typedef struct trunk_merge_stat
} TrunkMergeStat;
static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst,
FDFSTrunkFullInfo **ppLast, TrunkMergeStat *stat)
FDFSTrunkFullInfo **ppLast, TrunkMergeStat *merge_stat,
bool *bDeleted)
{
FDFSTrunkFullInfo **ppTrunkInfo;
int append_size;
char full_filename[MAX_PATH_SIZE];
struct stat file_stat;
(*ppMergeFirst)->file.size = (*ppLast)->file.offset -
(*ppMergeFirst)->file.offset + (*ppLast)->file.size;
stat->merge_count++;
stat->merged_trunk_count += (ppLast - ppMergeFirst) + 1;
stat->merged_size += (*ppMergeFirst)->file.size;
merge_stat->merge_count++;
merge_stat->merged_trunk_count += (ppLast - ppMergeFirst) + 1;
merge_stat->merged_size += (*ppMergeFirst)->file.size;
append_size = 0;
for (ppTrunkInfo=ppMergeFirst + 1; ppTrunkInfo<=ppLast; ppTrunkInfo++)
@ -563,6 +567,52 @@ static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst,
__sync_add_and_fetch(&g_trunk_total_free_space,
append_size);
do
{
if (!g_delete_unused_trunk_files)
{
break;
}
if (!((*ppMergeFirst)->file.offset == 0 &&
(*ppMergeFirst)->file.size >= g_trunk_file_size))
{
break;
}
trunk_get_full_filename(*ppMergeFirst, full_filename,
sizeof(full_filename));
if (stat(full_filename, &file_stat) != 0)
{
logError("file: "__FILE__", line: %d, "
"stat trunk file %s fail, "
"errno: %d, error info: %s", __LINE__,
full_filename, errno, STRERROR(errno));
break;
}
if ((*ppMergeFirst)->file.size != file_stat.st_size)
{
break;
}
if (unlink(full_filename) != 0)
{
if (errno != ENOENT)
{
logError("file: "__FILE__", line: %d, "
"unlink trunk file %s fail, "
"errno: %d, error info: %s", __LINE__,
full_filename, errno, STRERROR(errno));
break;
}
}
logInfo("file: "__FILE__", line: %d, "
"delete unused trunk file: %s",
__LINE__, full_filename);
trunk_delete_space_ex(*ppMergeFirst, false, false);
*bDeleted = true;
} while (0);
}
static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
@ -574,6 +624,7 @@ static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
TrunkMergeStat merge_stat;
char comma_buff[32];
int result;
bool bDeleted;
if (pCallbackArgs->trunk_array.count == 0)
{
@ -602,25 +653,35 @@ static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
continue;
}
bDeleted = false;
if (ppTrunkInfo - ppMergeFirst > 1)
{
trunk_merge_spaces(ppMergeFirst, previous, &merge_stat);
trunk_merge_spaces(ppMergeFirst, previous,
&merge_stat, &bDeleted);
}
if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
if (!bDeleted)
{
return result;
if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
{
return result;
}
}
ppMergeFirst = previous = ppTrunkInfo;
}
bDeleted = false;
if (ppEnd - ppMergeFirst > 1)
{
trunk_merge_spaces(ppMergeFirst, previous, &merge_stat);
trunk_merge_spaces(ppMergeFirst, previous,
&merge_stat, &bDeleted);
}
if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
if (!bDeleted)
{
return result;
if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
{
return result;
}
}
logInfo("file: "__FILE__", line: %d, "

1
storage/trunk_mgr/trunk_mem.h
View File

@ -58,6 +58,7 @@ extern bool g_trunk_create_file_advance;
extern bool g_trunk_init_check_occupying;
extern bool g_trunk_init_reload_from_binlog;
extern bool g_trunk_free_space_merge;
extern bool g_delete_unused_trunk_files;
extern int g_trunk_binlog_compress_stage;
extern bool g_if_trunker_self; //if am i trunk server
extern int64_t g_trunk_create_file_space_threshold;

5
tracker/tracker_func.c
View File

@ -601,6 +601,9 @@ int tracker_load_from_conf_file(const char *filename, \
g_trunk_free_space_merge = iniGetBoolValue(NULL,
"trunk_free_space_merge", &iniContext, false);
g_delete_unused_trunk_files = iniGetBoolValue(NULL,
"delete_unused_trunk_files", &iniContext, false);
if ((result=tracker_load_storage_id_info(
filename, &iniContext)) != 0)
{
@ -779,6 +782,7 @@ int tracker_load_from_conf_file(const char *filename, \
"trunk_init_check_occupying=%d, "
"trunk_init_reload_from_binlog=%d, "
"trunk_free_space_merge=%d, "
"delete_unused_trunk_files=%d, "
"trunk_compress_binlog_min_interval=%d, "
"trunk_compress_binlog_interval=%d, "
"trunk_compress_binlog_time_base=%02d:%02d, "
@ -823,6 +827,7 @@ int tracker_load_from_conf_file(const char *filename, \
(FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying,
g_trunk_init_reload_from_binlog,
g_trunk_free_space_merge,
g_delete_unused_trunk_files,
g_trunk_compress_binlog_min_interval,
g_trunk_compress_binlog_interval,
g_trunk_compress_binlog_time_base.hour,

1
tracker/tracker_global.c
View File

@ -52,6 +52,7 @@ bool g_trunk_create_file_advance = false;
bool g_trunk_init_check_occupying = false;
bool g_trunk_init_reload_from_binlog = false;
bool g_trunk_free_space_merge = false;
bool g_delete_unused_trunk_files = false;
int g_slot_min_size = 256; //slot min size, such as 256 bytes
int g_slot_max_size = 16 * 1024 * 1024; //slot max size, such as 16MB
int g_trunk_file_size = 64 * 1024 * 1024; //the trunk file size, such as 64MB

1
tracker/tracker_global.h
View File

@ -76,6 +76,7 @@ extern bool g_trunk_create_file_advance;
extern bool g_trunk_init_check_occupying;
extern bool g_trunk_init_reload_from_binlog;
extern bool g_trunk_free_space_merge;
extern bool g_delete_unused_trunk_files;
extern int g_slot_min_size; //slot min size, such as 256 bytes
extern int g_slot_max_size; //slot max size, such as 16MB
extern int g_trunk_file_size; //the trunk file size, such as 64MB

14
tracker/tracker_service.c
View File

@ -665,12 +665,12 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
if (pTask->length - sizeof(TrackerHeader) != 0)
{
logError("file: "__FILE__", line: %d, " \
"cmd=%d, client ip: %s, package size " \
PKG_LEN_PRINTF_FORMAT" is not correct, " \
"expect length = %d", __LINE__, \
TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ, \
pTask->client_ip, pTask->length - \
logError("file: "__FILE__", line: %d, "
"cmd=%d, client ip: %s, package size "
PKG_LEN_PRINTF_FORMAT" is not correct, "
"expect length = %d", __LINE__,
TRACKER_PROTO_CMD_STORAGE_PARAMETER_REQ,
pTask->client_ip, pTask->length -
(int)sizeof(TrackerHeader), 0);
pTask->length = sizeof(TrackerHeader);
@ -696,6 +696,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
"trunk_init_check_occupying=%d\n"
"trunk_init_reload_from_binlog=%d\n"
"trunk_free_space_merge=%d\n"
"delete_unused_trunk_files=%d\n"
"trunk_compress_binlog_min_interval=%d\n"
"trunk_compress_binlog_interval=%d\n"
"trunk_compress_binlog_time_base=%02d:%02d\n"
@ -718,6 +719,7 @@ static int tracker_deal_parameter_req(struct fast_task_info *pTask)
g_trunk_init_check_occupying,
g_trunk_init_reload_from_binlog,
g_trunk_free_space_merge,
g_delete_unused_trunk_files,
g_trunk_compress_binlog_min_interval,
g_trunk_compress_binlog_interval,
g_trunk_compress_binlog_time_base.hour,