pdxx
/
fastdfs
mirror of https://github.com/happyfish100/fastdfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bugfixed: delete first merged trunk node

pull/484/head
YuQing 2019-12-23 19:05:30 +08:00
parent b7447e5903
commit 71856858eb
2 changed files with 156 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
HISTORY
View File

@ -3,11 +3,13 @@ Version 6.05 2019-12-23
* fdfs_trackerd and fdfs_storaged print the server version in usage. * fdfs_trackerd and fdfs_storaged print the server version in usage.
you can execute fdfs_trackerd or fdfs_storaged without parameters you can execute fdfs_trackerd or fdfs_storaged without parameters
to show the server version to show the server version
* trunk server support compress the trunk binlog periodically, * trunk server support compress the trunk binlog periodically,
the config items in tracker.conf: trunk_compress_binlog_interval the config items in tracker.conf: trunk_compress_binlog_interval
and trunk_compress_binlog_time_base and trunk_compress_binlog_time_base
* trunk binlog compression support transaction * trunk binlog compression support transaction
* support backup binlog file when truncate trunk binlog, * support backup binlog file when truncate trunk binlog,
the config item in tracker.conf: trunk_binlog_max_backups the config item in tracker.conf: trunk_binlog_max_backups
@ -20,6 +22,7 @@ Version 6.05 2019-12-23
* support delete unused trunk files * support delete unused trunk files
the config item in tracker.conf: delete_unused_trunk_files the config item in tracker.conf: delete_unused_trunk_files
Version 6.04 2019-12-05 Version 6.04 2019-12-05
* storage_report_ip_changed ignore result EEXIST * storage_report_ip_changed ignore result EEXIST
* use get_gzip_command_filename from libfastcommon v1.42 * use get_gzip_command_filename from libfastcommon v1.42

279
storage/trunk_mgr/trunk_mem.c
View File

@ -81,7 +81,11 @@ static struct fast_mblock_man tree_nodes_man;
static AVLTreeInfo *tree_info_by_sizes = NULL; //for block alloc static AVLTreeInfo *tree_info_by_sizes = NULL; //for block alloc
static int trunk_create_next_file(FDFSTrunkFullInfo *pTrunkInfo); static int trunk_create_next_file(FDFSTrunkFullInfo *pTrunkInfo);
static int trunk_add_free_block(FDFSTrunkNode *pNode, const bool bWriteBinLog); static int trunk_add_free_block_ex(FDFSTrunkNode *pNode,
const bool bNeedLock, const bool bWriteBinLog);
#define trunk_add_free_block(pNode, bWriteBinLog) \
trunk_add_free_block_ex(pNode, true, bWriteBinLog)
static int trunk_restore_node(const FDFSTrunkFullInfo *pTrunkInfo); static int trunk_restore_node(const FDFSTrunkFullInfo *pTrunkInfo);
@ -90,6 +94,10 @@ static int trunk_delete_space_ex(const FDFSTrunkFullInfo *pTrunkInfo,
#define trunk_delete_space(pTrunkInfo, bWriteBinLog) \ #define trunk_delete_space(pTrunkInfo, bWriteBinLog) \
trunk_delete_space_ex(pTrunkInfo, true, bWriteBinLog) trunk_delete_space_ex(pTrunkInfo, true, bWriteBinLog)
static FDFSTrunkFullInfo *free_space_by_trunk(const FDFSTrunkFullInfo
*pTrunkInfo, const bool bNeedLock, const bool bWriteBinLog,
int *result);
static int storage_trunk_save(); static int storage_trunk_save();
static int storage_trunk_load(); static int storage_trunk_load();
@ -543,30 +551,25 @@ typedef struct trunk_merge_stat
} TrunkMergeStat; } TrunkMergeStat;
static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst, static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst,
FDFSTrunkFullInfo **ppLast, TrunkMergeStat *merge_stat, FDFSTrunkFullInfo **ppLast, TrunkMergeStat *merge_stat)
bool *bDeleted)
{ {
FDFSTrunkFullInfo **ppTrunkInfo; FDFSTrunkFullInfo **ppTrunkInfo;
int append_size; int merged_size;
char full_filename[MAX_PATH_SIZE]; char full_filename[MAX_PATH_SIZE];
struct stat file_stat; struct stat file_stat;
(*ppMergeFirst)->file.size = (*ppLast)->file.offset - merged_size = (*ppLast)->file.offset - (*ppMergeFirst)->file.offset
(*ppMergeFirst)->file.offset + (*ppLast)->file.size; + (*ppLast)->file.size;
merge_stat->merge_count++; merge_stat->merge_count++;
merge_stat->merged_trunk_count += (ppLast - ppMergeFirst) + 1; merge_stat->merged_trunk_count += (ppLast - ppMergeFirst) + 1;
merge_stat->merged_size += (*ppMergeFirst)->file.size; merge_stat->merged_size += merged_size;
append_size = 0;
for (ppTrunkInfo=ppMergeFirst + 1; ppTrunkInfo<=ppLast; ppTrunkInfo++) for (ppTrunkInfo=ppMergeFirst + 1; ppTrunkInfo<=ppLast; ppTrunkInfo++)
{ {
append_size += (*ppTrunkInfo)->file.size;
trunk_delete_space_ex(*ppTrunkInfo, false, false); trunk_delete_space_ex(*ppTrunkInfo, false, false);
} }
__sync_add_and_fetch(&g_trunk_total_free_space,
append_size);
do do
{ {
if (!g_delete_unused_trunk_files) if (!g_delete_unused_trunk_files)
@ -575,7 +578,7 @@ static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst,
} }
if (!((*ppMergeFirst)->file.offset == 0 && if (!((*ppMergeFirst)->file.offset == 0 &&
(*ppMergeFirst)->file.size >= g_trunk_file_size)) merged_size >= g_trunk_file_size))
{ {
break; break;
} }
@ -590,7 +593,7 @@ static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst,
full_filename, errno, STRERROR(errno)); full_filename, errno, STRERROR(errno));
break; break;
} }
if ((*ppMergeFirst)->file.size != file_stat.st_size) if (merged_size != file_stat.st_size)
{ {
break; break;
} }
@ -611,8 +614,21 @@ static void trunk_merge_spaces(FDFSTrunkFullInfo **ppMergeFirst,
"delete unused trunk file: %s", "delete unused trunk file: %s",
__LINE__, full_filename); __LINE__, full_filename);
trunk_delete_space_ex(*ppMergeFirst, false, false); trunk_delete_space_ex(*ppMergeFirst, false, false);
*bDeleted = true; *ppMergeFirst = NULL;
} while (0); } while (0);
if (*ppMergeFirst != NULL)
{
FDFSTrunkFullInfo trunkInfo;
int result;
trunkInfo = **ppMergeFirst;
trunkInfo.file.size = merged_size;
trunk_delete_space_ex(*ppMergeFirst, false, false);
*ppMergeFirst = free_space_by_trunk(&trunkInfo,
false, false, &result);
}
} }
static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs) static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
@ -624,7 +640,6 @@ static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
TrunkMergeStat merge_stat; TrunkMergeStat merge_stat;
char comma_buff[32]; char comma_buff[32];
int result; int result;
bool bDeleted;
if (pCallbackArgs->trunk_array.count == 0) if (pCallbackArgs->trunk_array.count == 0)
{ {
@ -653,13 +668,11 @@ static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
continue; continue;
} }
bDeleted = false;
if (ppTrunkInfo - ppMergeFirst > 1) if (ppTrunkInfo - ppMergeFirst > 1)
{ {
trunk_merge_spaces(ppMergeFirst, previous, trunk_merge_spaces(ppMergeFirst, previous, &merge_stat);
&merge_stat, &bDeleted);
} }
if (!bDeleted) if (*ppMergeFirst != NULL)
{ {
if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0) if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
{ {
@ -670,13 +683,11 @@ static int trunk_save_merged_spaces(struct walk_callback_args *pCallbackArgs)
ppMergeFirst = previous = ppTrunkInfo; ppMergeFirst = previous = ppTrunkInfo;
} }
bDeleted = false;
if (ppEnd - ppMergeFirst > 1) if (ppEnd - ppMergeFirst > 1)
{ {
trunk_merge_spaces(ppMergeFirst, previous, trunk_merge_spaces(ppMergeFirst, previous, &merge_stat);
&merge_stat, &bDeleted);
} }
if (!bDeleted) if (*ppMergeFirst != NULL)
{ {
if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0) if ((result=save_one_trunk(pCallbackArgs, *ppMergeFirst)) != 0)
{ {
@ -1672,48 +1683,23 @@ static int storage_trunk_load()
return storage_trunk_restore(restore_offset); return storage_trunk_restore(restore_offset);
} }
int trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo, \ static FDFSTrunkFullInfo *free_space_by_trunk(const FDFSTrunkFullInfo
const bool bWriteBinLog) *pTrunkInfo, const bool bNeedLock, const bool bWriteBinLog,
int *result)
{ {
int result;
struct fast_mblock_node *pMblockNode; struct fast_mblock_node *pMblockNode;
FDFSTrunkNode *pTrunkNode; FDFSTrunkNode *pTrunkNode;
if (!g_if_trunker_self)
{
logError("file: "__FILE__", line: %d, " \
"I am not trunk server!", __LINE__);
return EINVAL;
}
if (trunk_init_flag != STORAGE_TRUNK_INIT_FLAG_DONE)
{
if (bWriteBinLog)
{
logError("file: "__FILE__", line: %d, " \
"I am not inited!", __LINE__);
return EINVAL;
}
}
if (pTrunkInfo->file.size < g_slot_min_size)
{
logDebug("file: "__FILE__", line: %d, " \
"space: %d is too small, do not need recycle!", \
__LINE__, pTrunkInfo->file.size);
return 0;
}
pMblockNode = fast_mblock_alloc(&free_blocks_man); pMblockNode = fast_mblock_alloc(&free_blocks_man);
if (pMblockNode == NULL) if (pMblockNode == NULL)
{ {
result = errno != 0 ? errno : EIO; *result = errno != 0 ? errno : EIO;
logError("file: "__FILE__", line: %d, " \ logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail, " \ "malloc %d bytes fail, "
"errno: %d, error info: %s", \ "errno: %d, error info: %s",
__LINE__, (int)sizeof(FDFSTrunkNode), \ __LINE__, (int)sizeof(FDFSTrunkNode),
result, STRERROR(result)); *result, STRERROR(*result));
return result; return NULL;
} }
pTrunkNode = (FDFSTrunkNode *)pMblockNode->data; pTrunkNode = (FDFSTrunkNode *)pMblockNode->data;
@ -1722,89 +1708,130 @@ int trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo, \
pTrunkNode->pMblockNode = pMblockNode; pTrunkNode->pMblockNode = pMblockNode;
pTrunkNode->trunk.status = FDFS_TRUNK_STATUS_FREE; pTrunkNode->trunk.status = FDFS_TRUNK_STATUS_FREE;
pTrunkNode->next = NULL; pTrunkNode->next = NULL;
return trunk_add_free_block(pTrunkNode, bWriteBinLog); *result = trunk_add_free_block_ex(pTrunkNode, bNeedLock, bWriteBinLog);
return &pTrunkNode->trunk;
} }
static int trunk_add_free_block(FDFSTrunkNode *pNode, const bool bWriteBinLog) int trunk_free_space(const FDFSTrunkFullInfo *pTrunkInfo,
const bool bWriteBinLog)
{
int result;
if (!g_if_trunker_self)
{
logError("file: "__FILE__", line: %d, "
"I am not trunk server!", __LINE__);
return EINVAL;
}
if (trunk_init_flag != STORAGE_TRUNK_INIT_FLAG_DONE)
{
if (bWriteBinLog)
{
logError("file: "__FILE__", line: %d, "
"I am not inited!", __LINE__);
return EINVAL;
}
}
if (pTrunkInfo->file.size < g_slot_min_size)
{
logDebug("file: "__FILE__", line: %d, "
"space: %d is too small, do not need reclaim!",
__LINE__, pTrunkInfo->file.size);
return 0;
}
free_space_by_trunk(pTrunkInfo, true, bWriteBinLog, &result);
return result;
}
static int trunk_add_free_block_ex(FDFSTrunkNode *pNode,
const bool bNeedLock, const bool bWriteBinLog)
{ {
int result; int result;
struct fast_mblock_node *pMblockNode; struct fast_mblock_node *pMblockNode;
FDFSTrunkSlot target_slot; FDFSTrunkSlot target_slot;
FDFSTrunkSlot *chain; FDFSTrunkSlot *chain;
pthread_mutex_lock(&trunk_mem_lock); if (bNeedLock)
{
pthread_mutex_lock(&trunk_mem_lock);
}
if ((result=trunk_free_block_check_duplicate(&(pNode->trunk))) != 0) do
{ {
pthread_mutex_unlock(&trunk_mem_lock); if ((result=trunk_free_block_check_duplicate(&(pNode->trunk))) != 0)
return result; {
} break;
}
target_slot.size = pNode->trunk.file.size; target_slot.size = pNode->trunk.file.size;
target_slot.head = NULL; target_slot.head = NULL;
chain = (FDFSTrunkSlot *)avl_tree_find(tree_info_by_sizes + \ chain = (FDFSTrunkSlot *)avl_tree_find(tree_info_by_sizes +
pNode->trunk.path.store_path_index, &target_slot); pNode->trunk.path.store_path_index, &target_slot);
if (chain == NULL) if (chain == NULL)
{ {
pMblockNode = fast_mblock_alloc(&tree_nodes_man); pMblockNode = fast_mblock_alloc(&tree_nodes_man);
if (pMblockNode == NULL) if (pMblockNode == NULL)
{ {
result = errno != 0 ? errno : EIO; result = errno != 0 ? errno : EIO;
logError("file: "__FILE__", line: %d, " \ logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail, " \ "malloc %d bytes fail, "
"errno: %d, error info: %s", \ "errno: %d, error info: %s",
__LINE__, (int)sizeof(FDFSTrunkSlot), \ __LINE__, (int)sizeof(FDFSTrunkSlot),
result, STRERROR(result)); result, STRERROR(result));
pthread_mutex_unlock(&trunk_mem_lock); break;
return result; }
}
chain = (FDFSTrunkSlot *)pMblockNode->data; chain = (FDFSTrunkSlot *)pMblockNode->data;
chain->pMblockNode = pMblockNode; chain->pMblockNode = pMblockNode;
chain->size = pNode->trunk.file.size; chain->size = pNode->trunk.file.size;
pNode->next = NULL; pNode->next = NULL;
chain->head = pNode; chain->head = pNode;
if (avl_tree_insert(tree_info_by_sizes + pNode->trunk. \ if (avl_tree_insert(tree_info_by_sizes + pNode->trunk.
path.store_path_index, chain) != 1) path.store_path_index, chain) != 1)
{ {
result = errno != 0 ? errno : ENOMEM; result = errno != 0 ? errno : ENOMEM;
logError("file: "__FILE__", line: %d, " \ logError("file: "__FILE__", line: %d, "
"avl_tree_insert fail, " \ "avl_tree_insert fail, "
"errno: %d, error info: %s", \ "errno: %d, error info: %s",
__LINE__, result, STRERROR(result)); __LINE__, result, STRERROR(result));
pthread_mutex_unlock(&trunk_mem_lock); break;
return result; }
} }
} else
else {
{ pNode->next = chain->head;
pNode->next = chain->head; chain->head = pNode;
chain->head = pNode; }
}
if (bWriteBinLog) if (bWriteBinLog)
{ {
result = trunk_mem_binlog_write(g_current_time, result = trunk_mem_binlog_write(g_current_time,
TRUNK_OP_TYPE_ADD_SPACE, &(pNode->trunk)); TRUNK_OP_TYPE_ADD_SPACE, &(pNode->trunk));
} }
else else
{ {
__sync_add_and_fetch(&g_trunk_total_free_space, __sync_add_and_fetch(&g_trunk_total_free_space,
pNode->trunk.file.size); pNode->trunk.file.size);
result = 0; result = 0;
} }
if (result == 0) if (result == 0)
{ {
result = trunk_free_block_insert(&(pNode->trunk)); result = trunk_free_block_insert(&(pNode->trunk));
} }
else else
{ {
trunk_free_block_insert(&(pNode->trunk)); trunk_free_block_insert(&(pNode->trunk));
} }
} while (0);
pthread_mutex_unlock(&trunk_mem_lock); if (bNeedLock)
{
pthread_mutex_unlock(&trunk_mem_lock);
}
return result; return result;
} }