pdxx
/
fastdfs
mirror of https://github.com/happyfish100/fastdfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
fastdfs/storage/trunk_mgr/trunk_shared.c

769 lines
19 KiB
C
Raw Blame History

/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//trunk_shared.c
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include "fastcommon/logger.h"
#include "fastcommon/shared_func.h"
#include "trunk_shared.h"
#include "tracker_proto.h"
FDFSStorePaths g_fdfs_store_paths = {0, NULL};
struct base64_context g_fdfs_base64_context;
BufferInfo g_zero_buffer = {NULL, 0, 0};
int trunk_shared_init()
{
base64_init_ex(&g_fdfs_base64_context, 0, '-', '_', '.');
g_zero_buffer.alloc_size = g_zero_buffer.length = 256 * 1024;
g_zero_buffer.buff = (char *)malloc(g_zero_buffer.alloc_size);
if (g_zero_buffer.buff == NULL)
{
logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail", __LINE__,
g_zero_buffer.alloc_size);
return ENOMEM;
}
memset(g_zero_buffer.buff, 0, g_zero_buffer.length);
return 0;
}
FDFSStorePathInfo *storage_load_paths_from_conf_file_ex(
IniContext *pItemContext, const char *szSectionName,
const bool bUseBasePath, int *path_count, int *err_no)
{
char item_name[64];
FDFSStorePathInfo *store_paths;
char *pPath;
int bytes;
int i;
*path_count = iniGetIntValue(szSectionName, "store_path_count",
pItemContext, 1);
if (*path_count <= 0)
{
logError("file: "__FILE__", line: %d, "
"store_path_count: %d is invalid!",
__LINE__, *path_count);
*err_no = EINVAL;
return NULL;
}
bytes = sizeof(FDFSStorePathInfo) * (*path_count);
store_paths = (FDFSStorePathInfo *)malloc(bytes);
if (store_paths == NULL)
{
logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail, "
"errno: %d, error info: %s", __LINE__,
bytes, errno, STRERROR(errno));
*err_no = errno != 0 ? errno : ENOMEM;
return NULL;
}
memset(store_paths, 0, bytes);
pPath = iniGetStrValue(szSectionName, "store_path0", pItemContext);
if (pPath == NULL)
{
if (!bUseBasePath)
{
logError("file: "__FILE__", line: %d, "
"conf file must have item "
"\"store_path0\"!", __LINE__);
*err_no = ENOENT;
free(store_paths);
return NULL;
}
pPath = SF_G_BASE_PATH_STR;
}
store_paths[0].path_len = strlen(pPath);
store_paths[0].path = strdup(pPath);
if (store_paths[0].path == NULL)
{
logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail, "
"errno: %d, error info: %s",
__LINE__, (int)strlen(pPath),
errno, STRERROR(errno));
*err_no = errno != 0 ? errno : ENOMEM;
free(store_paths);
return NULL;
}
*err_no = 0;
for (i=1; i<*path_count; i++)
{
sprintf(item_name, "store_path%d", i);
pPath = iniGetStrValue(szSectionName, item_name,
pItemContext);
if (pPath == NULL)
{
logError("file: "__FILE__", line: %d, "
"conf file must have item \"%s\"!",
__LINE__, item_name);
*err_no = ENOENT;
break;
}
chopPath(pPath);
if (!fileExists(pPath))
{
logError("file: "__FILE__", line: %d, "
"\"%s\" can't be accessed, "
"errno: %d, error info: %s", __LINE__,
pPath, errno, STRERROR(errno));
*err_no = errno != 0 ? errno : ENOENT;
break;
}
if (!isDir(pPath))
{
logError("file: "__FILE__", line: %d, "
"\"%s\" is not a directory!",
__LINE__, pPath);
*err_no = ENOTDIR;
break;
}
store_paths[i].path_len = strlen(pPath);
store_paths[i].path = strdup(pPath);
if (store_paths[i].path == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, " \
"errno: %d, error info: %s", __LINE__, \
(int)strlen(pPath), errno, STRERROR(errno));
*err_no = errno != 0 ? errno : ENOMEM;
break;
}
}
if (*err_no != 0)
{
for (i=0; i<*path_count; i++)
{
if (store_paths[i].path != NULL)
{
free(store_paths[i].path);
}
}
free(store_paths);
return NULL;
}
return store_paths;
}
int storage_load_paths_from_conf_file(IniContext *pItemContext)
{
char *pPath;
int result;
pPath = iniGetStrValue(NULL, "base_path", pItemContext);
if (pPath == NULL)
{
logError("file: "__FILE__", line: %d, "
"conf file must have item \"base_path\"!", __LINE__);
return ENOENT;
}
snprintf(SF_G_BASE_PATH_STR, sizeof(SF_G_BASE_PATH_STR), "%s", pPath);
chopPath(SF_G_BASE_PATH_STR);
if (!fileExists(SF_G_BASE_PATH_STR))
{
logError("file: "__FILE__", line: %d, "
"\"%s\" can't be accessed, error info: %s",
__LINE__, STRERROR(errno), SF_G_BASE_PATH_STR);
return errno != 0 ? errno : ENOENT;
}
if (!isDir(SF_G_BASE_PATH_STR))
{
logError("file: "__FILE__", line: %d, "
"\"%s\" is not a directory!",
__LINE__, SF_G_BASE_PATH_STR);
return ENOTDIR;
}
g_fdfs_store_paths.paths = storage_load_paths_from_conf_file_ex(
pItemContext, NULL, true, &g_fdfs_store_paths.count, &result);
return result;
}
#define SPLIT_FILENAME_BODY(logic_filename, filename_len, true_filename, \
store_path_index, check_path_index) \
do \
{ \
char buff[3]; \
char *pEnd; \
\
if (*filename_len <= FDFS_LOGIC_FILE_PATH_LEN) \
{ \
logError("file: "__FILE__", line: %d, " \
"filename_len: %d is invalid, <= %d", \
__LINE__, *filename_len, FDFS_LOGIC_FILE_PATH_LEN); \
return EINVAL; \
} \
\
if (*logic_filename != FDFS_STORAGE_STORE_PATH_PREFIX_CHAR) \
{ /* version < V1.12 */ \
store_path_index = 0; \
memcpy(true_filename, logic_filename, (*filename_len)+1); \
break; \
} \
\
if (*(logic_filename + 3) != '/') \
{ \
logError("file: "__FILE__", line: %d, " \
"filename: %s is invalid", \
__LINE__, logic_filename); \
return EINVAL; \
} \
\
*buff = *(logic_filename+1); \
*(buff+1) = *(logic_filename+2); \
*(buff+2) = '\0'; \
\
pEnd = NULL; \
store_path_index = strtol(buff, &pEnd, 16); \
if (pEnd != NULL && *pEnd != '\0') \
{ \
logError("file: "__FILE__", line: %d, " \
"filename: %s is invalid", \
__LINE__, logic_filename); \
return EINVAL; \
} \
\
if (check_path_index && (store_path_index < 0 || \
store_path_index >= g_fdfs_store_paths.count)) \
{ \
logError("file: "__FILE__", line: %d, " \
"filename: %s is invalid, " \
"invalid store path index: %d", \
__LINE__, logic_filename, store_path_index); \
return EINVAL; \
} \
\
*filename_len -= 4; \
memcpy(true_filename, logic_filename + 4, (*filename_len) + 1); \
\
} while (0)
int storage_split_filename(const char *logic_filename, \
int *filename_len, char *true_filename, char **ppStorePath)
{
int store_path_index;
SPLIT_FILENAME_BODY(logic_filename, filename_len, true_filename, \
store_path_index, true);
*ppStorePath = g_fdfs_store_paths.paths[store_path_index].path;
return 0;
}
int storage_split_filename_ex(const char *logic_filename, \
int *filename_len, char *true_filename, int *store_path_index)
{
SPLIT_FILENAME_BODY(logic_filename, \
filename_len, true_filename, *store_path_index, true);
return 0;
}
int storage_split_filename_no_check(const char *logic_filename, \
int *filename_len, char *true_filename, int *store_path_index)
{
SPLIT_FILENAME_BODY(logic_filename, \
filename_len, true_filename, *store_path_index, false);
return 0;
}
char *trunk_info_dump(const FDFSTrunkFullInfo *pTrunkInfo, char *buff, \
const int buff_size)
{
snprintf(buff, buff_size, \
"store_path_index=%d, " \
"sub_path_high=%d, " \
"sub_path_low=%d, " \
"id=%u, offset=%d, size=%d, status=%d", \
pTrunkInfo->path.store_path_index, \
pTrunkInfo->path.sub_path_high, \
pTrunkInfo->path.sub_path_low, \
pTrunkInfo->file.id, pTrunkInfo->file.offset, pTrunkInfo->file.size, \
pTrunkInfo->status);
return buff;
}
char *trunk_header_dump(const FDFSTrunkHeader *pTrunkHeader, char *buff, \
const int buff_size)
{
snprintf(buff, buff_size, \
"file_type=%d, " \
"alloc_size=%d, " \
"file_size=%d, " \
"crc32=%d, " \
"mtime=%d, " \
"ext_name(%d)=%s", \
pTrunkHeader->file_type, pTrunkHeader->alloc_size, \
pTrunkHeader->file_size, pTrunkHeader->crc32, \
pTrunkHeader->mtime, \
(int)strlen(pTrunkHeader->formatted_ext_name), \
pTrunkHeader->formatted_ext_name);
return buff;
}
char *trunk_get_full_filename_ex(const FDFSStorePaths *pStorePaths, \
const FDFSTrunkFullInfo *pTrunkInfo, \
char *full_filename, const int buff_size)
{
char short_filename[64];
char *pStorePath;
pStorePath = pStorePaths->paths[pTrunkInfo->path.store_path_index].path;
TRUNK_GET_FILENAME(pTrunkInfo->file.id, short_filename);
snprintf(full_filename, buff_size, \
"%s/data/"FDFS_STORAGE_DATA_DIR_FORMAT"/" \
FDFS_STORAGE_DATA_DIR_FORMAT"/%s", \
pStorePath, pTrunkInfo->path.sub_path_high, \
pTrunkInfo->path.sub_path_low, short_filename);
return full_filename;
}
void trunk_pack_header(const FDFSTrunkHeader *pTrunkHeader, char *buff)
{
*(buff + FDFS_TRUNK_FILE_FILE_TYPE_OFFSET) = pTrunkHeader->file_type;
int2buff(pTrunkHeader->alloc_size, \
buff + FDFS_TRUNK_FILE_ALLOC_SIZE_OFFSET);
int2buff(pTrunkHeader->file_size, \
buff + FDFS_TRUNK_FILE_FILE_SIZE_OFFSET);
int2buff(pTrunkHeader->crc32, \
buff + FDFS_TRUNK_FILE_FILE_CRC32_OFFSET);
int2buff(pTrunkHeader->mtime, \
buff + FDFS_TRUNK_FILE_FILE_MTIME_OFFSET);
memcpy(buff + FDFS_TRUNK_FILE_FILE_EXT_NAME_OFFSET, \
pTrunkHeader->formatted_ext_name, \
FDFS_FILE_EXT_NAME_MAX_LEN + 1);
}
void trunk_unpack_header(const char *buff, FDFSTrunkHeader *pTrunkHeader)
{
pTrunkHeader->file_type = *(buff + FDFS_TRUNK_FILE_FILE_TYPE_OFFSET);
pTrunkHeader->alloc_size = buff2int(
buff + FDFS_TRUNK_FILE_ALLOC_SIZE_OFFSET);
pTrunkHeader->file_size = buff2int(
buff + FDFS_TRUNK_FILE_FILE_SIZE_OFFSET);
pTrunkHeader->crc32 = buff2int(
buff + FDFS_TRUNK_FILE_FILE_CRC32_OFFSET);
pTrunkHeader->mtime = buff2int(
buff + FDFS_TRUNK_FILE_FILE_MTIME_OFFSET);
memcpy(pTrunkHeader->formatted_ext_name, buff + \
FDFS_TRUNK_FILE_FILE_EXT_NAME_OFFSET, \
FDFS_FILE_EXT_NAME_MAX_LEN + 1);
*(pTrunkHeader->formatted_ext_name+FDFS_FILE_EXT_NAME_MAX_LEN+1)='\0';
}
void trunk_file_info_encode(const FDFSTrunkFileInfo *pTrunkFile, char *str)
{
char buff[sizeof(int) * 3];
int len;
int2buff(pTrunkFile->id, buff);
int2buff(pTrunkFile->offset, buff + sizeof(int));
int2buff(pTrunkFile->size, buff + sizeof(int) * 2);
base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(buff),
str, &len, false);
}
void trunk_file_info_decode(const char *str, FDFSTrunkFileInfo *pTrunkFile)
{
char buff[FDFS_TRUNK_FILE_INFO_LEN];
int len;
base64_decode_auto(&g_fdfs_base64_context, str, FDFS_TRUNK_FILE_INFO_LEN,
buff, &len);
pTrunkFile->id = buff2int(buff);
pTrunkFile->offset = buff2int(buff + sizeof(int));
pTrunkFile->size = buff2int(buff + sizeof(int) * 2);
}
int trunk_file_get_content_ex(const FDFSStorePaths *pStorePaths, \
const FDFSTrunkFullInfo *pTrunkInfo, const int file_size, \
int *pfd, char *buff, const int buff_size)
{
char full_filename[MAX_PATH_SIZE];
int fd;
int result;
int read_bytes;
if (file_size > buff_size)
{
return ENOSPC;
}
if (pfd != NULL)
{
fd = *pfd;
}
else
{
trunk_get_full_filename_ex(pStorePaths, pTrunkInfo, \
full_filename, sizeof(full_filename));
fd = open(full_filename, O_RDONLY);
if (fd < 0)
{
return errno != 0 ? errno : EIO;
}
if (lseek(fd, pTrunkInfo->file.offset + \
FDFS_TRUNK_FILE_HEADER_SIZE, SEEK_SET) < 0)
{
result = errno != 0 ? errno : EIO;
close(fd);
return result;
}
}
read_bytes = fc_safe_read(fd, buff, file_size);
if (read_bytes == file_size)
{
result = 0;
}
else
{
result = errno != 0 ? errno : EINVAL;
}
if (pfd == NULL)
{
close(fd);
}
return result;
}
int trunk_file_stat_func_ex(const FDFSStorePaths *pStorePaths, \
const int store_path_index, const char *true_filename, \
const int filename_len, const int stat_func, \
struct stat *pStat, FDFSTrunkFullInfo *pTrunkInfo, \
FDFSTrunkHeader *pTrunkHeader, int *pfd)
{
int result;
int src_store_path_index;
int src_filename_len;
char src_filename[128];
char src_true_filename[128];
result = trunk_file_do_lstat_func_ex(pStorePaths, store_path_index, \
true_filename, filename_len, stat_func, \
pStat, pTrunkInfo, pTrunkHeader, pfd);
if (result != 0)
{
return result;
}
if (!(stat_func == FDFS_STAT_FUNC_STAT && IS_TRUNK_FILE_BY_ID( \
(*pTrunkInfo)) && S_ISLNK(pStat->st_mode)))
{
return 0;
}
do
{
result = trunk_file_get_content_ex(pStorePaths, pTrunkInfo, \
pStat->st_size, pfd, src_filename, \
sizeof(src_filename) - 1);
if (result != 0)
{
break;
}
src_filename_len = pStat->st_size;
*(src_filename + src_filename_len) = '\0';
if ((result=storage_split_filename_no_check(src_filename, \
&src_filename_len, src_true_filename, \
&src_store_path_index)) != 0)
{
break;
}
if (src_store_path_index < 0 || \
src_store_path_index >= pStorePaths->count)
{
logError("file: "__FILE__", line: %d, " \
"filename: %s is invalid, " \
"invalid store path index: %d, " \
"which < 0 or >= %d", __LINE__, \
src_filename, src_store_path_index, \
pStorePaths->count);
result = EINVAL;
break;
}
if (pfd != NULL)
{
close(*pfd);
*pfd = -1;
}
result = trunk_file_do_lstat_func_ex(pStorePaths, \
src_store_path_index, src_true_filename, \
src_filename_len, stat_func, pStat, \
pTrunkInfo, pTrunkHeader, pfd);
} while (0);
if (result != 0 && pfd != NULL && *pfd >= 0)
{
close(*pfd);
*pfd = -1;
}
return result;
}
int trunk_file_do_lstat_func_ex(const FDFSStorePaths *pStorePaths, \
const int store_path_index, const char *true_filename, \
const int filename_len, const int stat_func, \
struct stat *pStat, FDFSTrunkFullInfo *pTrunkInfo, \
FDFSTrunkHeader *pTrunkHeader, int *pfd)
{
char full_filename[MAX_PATH_SIZE];
char buff[128];
char pack_buff[FDFS_TRUNK_FILE_HEADER_SIZE];
int64_t file_size;
int buff_len;
int fd;
int read_bytes;
int result;
pTrunkInfo->file.id = 0;
if (filename_len != FDFS_TRUNK_FILENAME_LENGTH) //not trunk file
{
snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \
pStorePaths->paths[store_path_index].path, true_filename);
if (stat_func == FDFS_STAT_FUNC_STAT)
{
result = stat(full_filename, pStat);
}
else
{
result = lstat(full_filename, pStat);
}
if (result == 0)
{
return 0;
}
else
{
return errno != 0 ? errno : ENOENT;
}
}
memset(buff, 0, sizeof(buff));
base64_decode_auto(&g_fdfs_base64_context, (char *)true_filename + \
FDFS_TRUE_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \
buff, &buff_len);
file_size = buff2long(buff + sizeof(int) * 2);
if (!IS_TRUNK_FILE(file_size)) //slave file
{
snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \
pStorePaths->paths[store_path_index].path, true_filename);
if (stat_func == FDFS_STAT_FUNC_STAT)
{
result = stat(full_filename, pStat);
}
else
{
result = lstat(full_filename, pStat);
}
if (result == 0)
{
return 0;
}
else
{
return errno != 0 ? errno : ENOENT;
}
}
trunk_file_info_decode(true_filename + FDFS_TRUE_FILE_PATH_LEN + \
FDFS_FILENAME_BASE64_LENGTH, &pTrunkInfo->file);
pTrunkHeader->file_size = FDFS_TRUNK_FILE_TRUE_SIZE(file_size);
pTrunkHeader->mtime = buff2int(buff + sizeof(int));
pTrunkHeader->crc32 = buff2int(buff + sizeof(int) * 4);
memcpy(pTrunkHeader->formatted_ext_name, true_filename + \
(filename_len - (FDFS_FILE_EXT_NAME_MAX_LEN + 1)), \
FDFS_FILE_EXT_NAME_MAX_LEN + 2); //include tailing '\0'
pTrunkHeader->alloc_size = pTrunkInfo->file.size;
pTrunkInfo->path.store_path_index = store_path_index;
pTrunkInfo->path.sub_path_high = strtol(true_filename, NULL, 16);
pTrunkInfo->path.sub_path_low = strtol(true_filename + 3, NULL, 16);
trunk_get_full_filename_ex(pStorePaths, pTrunkInfo, full_filename, \
sizeof(full_filename));
fd = open(full_filename, O_RDONLY);
if (fd < 0)
{
return errno != 0 ? errno : EIO;
}
if (lseek(fd, pTrunkInfo->file.offset, SEEK_SET) < 0)
{
result = errno != 0 ? errno : EIO;
close(fd);
return result;
}
read_bytes = fc_safe_read(fd, buff, FDFS_TRUNK_FILE_HEADER_SIZE);
if (read_bytes == FDFS_TRUNK_FILE_HEADER_SIZE)
{
result = 0;
}
else
{
result = errno;
close(fd);
return result != 0 ? result : EINVAL;
}
memset(pStat, 0, sizeof(struct stat));
pTrunkHeader->file_type = *(buff + FDFS_TRUNK_FILE_FILE_TYPE_OFFSET);
if (pTrunkHeader->file_type == FDFS_TRUNK_FILE_TYPE_REGULAR)
{
pStat->st_mode = S_IFREG;
}
else if (pTrunkHeader->file_type == FDFS_TRUNK_FILE_TYPE_LINK)
{
pStat->st_mode = S_IFLNK;
}
else if (pTrunkHeader->file_type == FDFS_TRUNK_FILE_TYPE_NONE)
{
close(fd);
return ENOENT;
}
else
{
/*
logError("file: "__FILE__", line: %d, "
"Invalid file type: %d", __LINE__,
pTrunkHeader->file_type);
*/
close(fd);
return ENOENT;
}
trunk_pack_header(pTrunkHeader, pack_buff);
/*
{
char temp[265];
char szHexBuff[2 * FDFS_TRUNK_FILE_HEADER_SIZE + 1];
FDFSTrunkHeader trueTrunkHeader;
fprintf(stderr, "file: "__FILE__", line: %d, true buff=%s\n", __LINE__, \
bin2hex(buff+1, FDFS_TRUNK_FILE_HEADER_SIZE - 1, szHexBuff));
trunk_unpack_header(buff, &trueTrunkHeader);
fprintf(stderr, "file: "__FILE__", line: %d, true fields=%s\n", __LINE__, \
trunk_header_dump(&trueTrunkHeader, full_filename, sizeof(full_filename)));
fprintf(stderr, "file: "__FILE__", line: %d, my buff=%s\n", __LINE__, \
bin2hex(pack_buff+1, FDFS_TRUNK_FILE_HEADER_SIZE - 1, szHexBuff));
fprintf(stderr, "file: "__FILE__", line: %d, my trunk=%s, my fields=%s\n", __LINE__, \
trunk_info_dump(pTrunkInfo, temp, sizeof(temp)), \
trunk_header_dump(pTrunkHeader, full_filename, sizeof(full_filename)));
}
*/
if (memcmp(pack_buff, buff, FDFS_TRUNK_FILE_HEADER_SIZE) != 0)
{
close(fd);
return ENOENT;
}
pStat->st_size = pTrunkHeader->file_size;
pStat->st_mtime = pTrunkHeader->mtime;
if (pfd != NULL)
{
*pfd = fd;
}
else
{
close(fd);
}
return 0;
}
bool fdfs_is_trunk_file(const char *remote_filename, const int filename_len)
{
int buff_len;
char buff[64];
int64_t file_size;
if (filename_len != FDFS_TRUNK_LOGIC_FILENAME_LENGTH) //not trunk file
{
return false;
}
memset(buff, 0, sizeof(buff));
base64_decode_auto(&g_fdfs_base64_context, (char *)remote_filename + \
FDFS_LOGIC_FILE_PATH_LEN, FDFS_FILENAME_BASE64_LENGTH, \
buff, &buff_len);
file_size = buff2long(buff + sizeof(int) * 2);
return IS_TRUNK_FILE(file_size);
}
int fdfs_decode_trunk_info(const int store_path_index, \
const char *true_filename, const int filename_len, \
FDFSTrunkFullInfo *pTrunkInfo)
{
if (filename_len != FDFS_TRUNK_FILENAME_LENGTH) //not trunk file
{
logWarning("file: "__FILE__", line: %d, " \
"trunk filename length: %d != %d, filename: %s", \
__LINE__, filename_len, FDFS_TRUNK_FILENAME_LENGTH, \
true_filename);
return EINVAL;
}
pTrunkInfo->path.store_path_index = store_path_index;
pTrunkInfo->path.sub_path_high = strtol(true_filename, NULL, 16);
pTrunkInfo->path.sub_path_low = strtol(true_filename + 3, NULL, 16);
trunk_file_info_decode(true_filename + FDFS_TRUE_FILE_PATH_LEN + \
FDFS_FILENAME_BASE64_LENGTH, &pTrunkInfo->file);
return 0;
}
Reference in New Issue View Git Blame Copy Permalink