pdxx
/
fastdfs
mirror of https://github.com/happyfish100/fastdfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 55b2eeafc1 into f151ca8c42

pull/484/merge
songqingchao 2021-01-10 15:23:52 +08:00 committed by GitHub
parent f151ca8c42 55b2eeafc1
commit 4945a963c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
126 changed files with 8402 additions and 3356 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
.gitignore vendored Normal file
View File

@ -0,0 +1,106 @@
# Makefile.in
storage/Makefile
tracker/Makefile
client/test/Makefile
client/Makefile
# client/fdfs_link_library.sh.in
client/fdfs_link_library.sh
# Prerequisites
*.d
# Compiled Object files
*.slo
*.lo
*.o
*.obj
# Precompiled Headers
*.gch
*.pch
# Compiled Dynamic libraries
*.so
*.dylib
*.dSYM
*.dll
# Fortran module files
*.mod
*.smod
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
# Executables
*.exe
*.out
*.app
client/fdfs_append_file
client/fdfs_appender_test
client/fdfs_appender_test1
client/fdfs_crc32
client/fdfs_delete_file
client/fdfs_download_file
client/fdfs_file_info
client/fdfs_monitor
client/fdfs_test
client/fdfs_test1
client/fdfs_upload_appender
client/fdfs_upload_file
client/fdfs_regenerate_filename
client/test/fdfs_monitor
client/test/fdfs_test
client/test/fdfs_test1
storage/fdfs_storaged
tracker/fdfs_trackerd
test/combine_result
test/100M
test/10M
test/1M
test/200K
test/50K
test/5K
test/gen_files
test/test_delete
test/test_download
test/test_upload
test/upload/
test/download/
test/delete/
# other
php_client/.deps
php_client/.libs/
php_client/Makefile
php_client/Makefile.fragments
php_client/Makefile.global
php_client/Makefile.objects
php_client/acinclude.m4
php_client/aclocal.m4
php_client/autom4te.cache/
php_client/build/
php_client/config.guess
php_client/config.h
php_client/config.h.in
php_client/config.log
php_client/config.nice
php_client/config.status
php_client/config.sub
php_client/configure
php_client/configure.ac
php_client/install-sh
php_client/libtool
php_client/ltmain.sh
php_client/missing
php_client/mkinstalldirs
php_client/run-tests.php
# fastdfs runtime paths
data/
logs/

84
HISTORY
View File

@ -1,7 +1,88 @@
Version 6.07 2020-09-30
* use libfastcommon V1.44
NOTE: you MUST upgrade libfastcommon to V1.44 or later
* correct spell iovent to ioevent follows libfastcommon
Version 6.06 2019-12-30
* bugfixed: fdfs_storaged can't quit normally
* bugfixed: init/memset return ip address to ascii 0 for Java SDK
Version 6.05 2019-12-25
* fdfs_trackerd and fdfs_storaged print the server version in usage.
you can execute fdfs_trackerd or fdfs_storaged without parameters
to show the server version
* trunk server support compress the trunk binlog periodically,
the config items in tracker.conf: trunk_compress_binlog_interval
and trunk_compress_binlog_time_base
* trunk binlog compression support transaction
* support backup binlog file when truncate trunk binlog,
the config item in tracker.conf: trunk_binlog_max_backups
* support alignment size for trunk space allocation
the config item in tracker.conf: trunk_alloc_alignment_size
* support merge free trunk spaces
the config item in tracker.conf: trunk_free_space_merge
* support delete unused trunk files
the config item in tracker.conf: delete_unused_trunk_files
* fdfs_monitor.c: do NOT call getHostnameByIp
NOTE: you MUST upgrade libfastcommon to V1.43 or later
Version 6.04 2019-12-05
* storage_report_ip_changed ignore result EEXIST
* use get_gzip_command_filename from libfastcommon v1.42
* support compress error log and access log
* disk recovery support multi-threads to speed up
* bugfix: should use memset to init pReader in function
storage_reader_init, this bug is caused by v6.01
NOTE: you MUST upgrade libfastcommon to V1.42 or later
Version 6.03 2019-11-20
* dual IPs support two different types of inner (intranet) IPs
* storage server request tracker server to change it's status
to that of tracker leader when the storage server found
it's status inconsistence
* bugfix: fdfs_monitor fix get index of the specified tracker server
* storage server write to data_init_flag and mark file safely
(write to temp file then rename)
* code refine: combine g_fdfs_store_paths and g_path_space_list,
and extent struct FDFSStorePathInfo
* check store path's mark file to prevent confusion
* new selected tracker leader do NOT notify self by network
* larger network_timeout for fetching one-store-path binlog
when disk recovery
NOTE: the tracker and storage server must upgrade together
Version 6.02 2019-11-12
* get_file_info calculate CRC32 for appender file type
* disk recovery download file to local temp file then rename it
when the local file exists
* support regenerate filename for appender file
NOTE: the regenerated file will be a normal file!
Version 6.01 2019-10-25
* compress and uncompress binlog file by gzip when need,
config items in storage.conf: compress_binlog and compress_binlog_time
* bugfix: must check and create data path before write_to_pid_file
in fdfs_storaged.c
Version 6.00 2019-10-16
* tracker and storage server support dual IPs
1. you can config dual trackr IPs in storage.conf and client.conf,
1. you can config dual tracker IPs in storage.conf and client.conf,
the configuration item name is "tracker_server"
2. you can config dual storage IPs in storage_ids.conf
more detail please see the config files.
@ -14,6 +95,7 @@ Version 6.00 2019-10-16
* tracker server check tracker list when storage server join
* use socketCreateExAuto and socketClientExAuto exported by libfastcommon
Version 5.12 2018-06-07
* code refine for rare case
* replace print format OFF_PRINTF_FORMAT to PRId64

75
INSTALL
View File

@ -3,50 +3,68 @@ Copy right 2009 Happy Fish / YuQing
FastDFS may be copied only under the terms of the GNU General
Public License V3, which may be found in the FastDFS source kit.
Please visit the FastDFS Home Page for more detail.
English language: http://english.csource.org/
Chinese language: http://www.csource.org/
Chinese language: http://www.fastken.com/
#step 1. download libfastcommon source package from github and install it,
the github address:
https://github.com/happyfish100/libfastcommon.git
# step 1. download libfastcommon source codes and install it,
# github address: https://github.com/happyfish100/libfastcommon.git
# gitee address: https://gitee.com/fastdfs100/libfastcommon.git
# command lines as:
#step 2. download FastDFS source package and unpack it,
tar xzf FastDFS_v5.x.tar.gz
#for example:
tar xzf FastDFS_v5.08.tar.gz
git clone https://github.com/happyfish100/libfastcommon.git
cd libfastcommon; git checkout V1.0.43
./make.sh clean && ./make.sh && ./make.sh install
#step 3. enter the FastDFS dir
cd FastDFS
#step 4. execute:
./make.sh
# step 2. download fastdfs source codes and install it,
# github address: https://github.com/happyfish100/fastdfs.git
# gitee address: https://gitee.com/fastdfs100/fastdfs.git
# command lines as:
#step 5. make install
./make.sh install
git clone https://github.com/happyfish100/fastdfs.git
cd fastdfs; git checkout V6.06
./make.sh clean && ./make.sh && ./make.sh install
#step 6. edit/modify the config file of tracker and storage
#step 7. run server programs
# step 3. setup the config files
# the setup script does NOT overwrite existing config files,
# please feel free to execute this script (take easy :)
./setup.sh /etc/fdfs
# step 4. edit or modify the config files of tracker, storage and client
such as:
vi /etc/fdfs/tracker.conf
vi /etc/fdfs/storage.conf
vi /etc/fdfs/client.conf
and so on ...
# step 5. run the server programs
# start the tracker server:
/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf restart
#in Linux, you can start fdfs_trackerd as a service:
/sbin/service fdfs_trackerd start
# start the storage server:
/usr/bin/fdfs_storaged /etc/fdfs/storage.conf restart
#in Linux, you can start fdfs_storaged as a service:
/sbin/service fdfs_storaged start
#step 8. run test program
#run the client test program:
# (optional) in Linux, you can start fdfs_trackerd and fdfs_storaged as a service:
/sbin/service fdfs_trackerd restart
/sbin/service fdfs_storaged restart
# step 6. (optional) run monitor program
# such as:
/usr/bin/fdfs_monitor /etc/fdfs/client.conf
# step 7. (optional) run the test program
# such as:
/usr/bin/fdfs_test <client_conf_filename> <operation>
/usr/bin/fdfs_test1 <client_conf_filename> <operation>
#for example, upload a file:
/usr/bin/fdfs_test conf/client.conf upload /usr/include/stdlib.h
#step 9. run monitor program
#run the monitor program:
/usr/bin/fdfs_monitor <client_conf_filename>
# for example, upload a file for test:
/usr/bin/fdfs_test /etc/fdfs/client.conf upload /usr/include/stdlib.h
tracker server config file sample please see conf/tracker.conf
@ -55,7 +73,6 @@ storage server config file sample please see conf/storage.conf
client config file sample please see conf/client.conf
Item detail
1. server common items
---------------------------------------------------

6
README.md
View File

@ -3,8 +3,7 @@ Copyright (C) 2008 Happy Fish / YuQing
FastDFS may be copied only under the terms of the GNU General
Public License V3, which may be found in the FastDFS source kit.
Please visit the FastDFS Home Page for more detail.
English language: http://english.csource.org/
Chinese language: http://www.csource.org/
Chinese language: http://www.fastken.com/
FastDFS is an open source high performance distributed file system. It's major
@ -42,3 +41,6 @@ The identification of a file is composed of two parts: the volume name and
the file name.
Client test code use client library please refer to the directory: client/test.
For more FastDFS related articles, please subscribe the Wechat/Weixin public account
(Chinese Language): fastdfs

20
README_zh.md Normal file
View File

@ -0,0 +1,20 @@
FastDFS是一款开源的分布式文件系统，功能主要包括：文件存储、文件同步、文件访问（文件上传、文件下载）等，解决了文件大容量存储和高性能访问的问题。FastDFS特别适合以文件为载体的在线服务，如图片、视频、文档等等。
FastDFS作为一款轻量级分布式文件系统，版本V6.01代码量6.3万行。FastDFS用C语言实现，支持Linux、FreeBSD、MacOS等类UNIX系统。FastDFS类似google FS，属于应用级文件系统，不是通用的文件系统，只能通过专有API访问，目前提供了C和Java SDK，以及PHP扩展SDK。
FastDFS为互联网应用量身定做，解决大容量文件存储问题，追求高性能和高扩展性。FastDFS可以看做是基于文件的key value存储系统，key为文件ID，value为文件内容，因此称作分布式文件存储服务更为合适。
FastDFS的架构比较简单，如下图所示：
![architect](images/architect.png)
FastDFS特点如下：
1）分组存储，简单灵活；
2）对等结构，不存在单点；
3）文件ID由FastDFS生成，作为文件访问凭证。FastDFS不需要传统的name server或meta server；
4）大、中、小文件均可以很好支持，可以存储海量小文件；
5）一台storage支持多块磁盘，支持单盘数据恢复；
6）提供了nginx扩展模块，可以和nginx无缝衔接；
7）支持多线程方式上传和下载文件，支持断点续传；
8）存储服务器上可以保存文件附加属性。
FastDFS更多更详细的功能和特性介绍，请参阅FastDFS微信公众号的其他文章，搜索公众号：fastdfs。

4
client/Makefile.in
View File

@ -6,7 +6,7 @@ ENABLE_SHARED_LIB = $(ENABLE_SHARED_LIB)
INC_PATH = -I../common -I../tracker -I/usr/include/fastcommon
LIB_PATH = $(LIBS) -lfastcommon
TARGET_PATH = $(TARGET_PREFIX)/bin
TARGET_LIB = $(TARGET_PREFIX)/lib64
TARGET_LIB = $(TARGET_PREFIX)/$(LIB_VERSION)
TARGET_INC = $(TARGET_PREFIX)/include
CONFIG_PATH = $(TARGET_CONF_PATH)
@ -39,7 +39,7 @@ ALL_OBJS = $(STATIC_OBJS) $(FDFS_SHARED_OBJS)
ALL_PRGS = fdfs_monitor fdfs_test fdfs_test1 fdfs_crc32 fdfs_upload_file \
fdfs_download_file fdfs_delete_file fdfs_file_info \
fdfs_appender_test fdfs_appender_test1 fdfs_append_file \
fdfs_upload_appender
fdfs_upload_appender fdfs_regenerate_filename
STATIC_LIBS = libfdfsclient.a

2
client/client_func.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//client_func.c

4
client/client_func.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//client_func.h
@ -16,6 +16,8 @@
#define _CLIENT_FUNC_H_
typedef struct {
short file_type;
bool get_from_server;
time_t create_timestamp;
int crc32;
int source_id; //source storage id

2
client/client_global.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdlib.h>

2
client/client_global.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//client_global.h

2
client/fdfs_append_file.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>

4
client/fdfs_appender_test.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -83,7 +83,7 @@ int main(int argc, char *argv[])
"\nCopyright (C) 2008, Happy Fish / YuQing\n" \
"\nFastDFS may be copied only under the terms of the GNU General\n" \
"Public License V3, which may be found in the FastDFS source kit.\n" \
"Please visit the FastDFS Home Page http://www.csource.org/ \n" \
"Please visit the FastDFS Home Page http://www.fastken.com/ \n" \
"for more detail.\n\n" \
, g_fdfs_version.major, g_fdfs_version.minor);

4
client/fdfs_appender_test1.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -82,7 +82,7 @@ int main(int argc, char *argv[])
"\nCopyright (C) 2008, Happy Fish / YuQing\n" \
"\nFastDFS may be copied only under the terms of the GNU General\n" \
"Public License V3, which may be found in the FastDFS source kit.\n" \
"Please visit the FastDFS Home Page http://www.csource.org/ \n" \
"Please visit the FastDFS Home Page http://www.fastken.com/ \n" \
"for more detail.\n\n" \
, g_fdfs_version.major, g_fdfs_version.minor);

2
client/fdfs_client.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#ifndef FDFS_CLIENT_H

2
client/fdfs_crc32.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>

2
client/fdfs_delete_file.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>

2
client/fdfs_download_file.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>

26
client/fdfs_file_info.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -19,6 +19,7 @@
int main(int argc, char *argv[])
{
char *conf_filename;
const char *file_type_str;
char file_id[128];
int result;
FDFSFileInfo file_info;
@ -44,7 +45,7 @@ int main(int argc, char *argv[])
result = fdfs_get_file_info_ex1(file_id, true, &file_info);
if (result != 0)
{
printf("query file info fail, " \
fprintf(stderr, "query file info fail, " \
"error no: %d, error info: %s\n", \
result, STRERROR(result));
}
@ -52,6 +53,25 @@ int main(int argc, char *argv[])
{
char szDatetime[32];
switch (file_info.file_type)
{
case FDFS_FILE_TYPE_NORMAL:
file_type_str = "normal";
break;
case FDFS_FILE_TYPE_SLAVE:
file_type_str = "slave";
break;
case FDFS_FILE_TYPE_APPENDER:
file_type_str = "appender";
break;
default:
file_type_str = "unkown";
break;
}
printf("GET FROM SERVER: %s\n\n",
file_info.get_from_server ? "true" : "false");
printf("file type: %s\n", file_type_str);
printf("source storage id: %d\n", file_info.source_id);
printf("source ip address: %s\n", file_info.source_ip_addr);
printf("file create timestamp: %s\n", formatDatetime(
@ -59,7 +79,7 @@ int main(int argc, char *argv[])
szDatetime, sizeof(szDatetime)));
printf("file size: %"PRId64"\n", \
file_info.file_size);
printf("file crc32: %u (0x%08X)\n", \
printf("file crc32: %d (0x%08x)\n", \
file_info.crc32, file_info.crc32);
}

328
client/fdfs_monitor.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -131,9 +131,9 @@ int main(int argc, char *argv[])
for (i=0; i<g_tracker_group.server_count; i++)
{
if (fdfs_server_contain1(g_tracker_group.servers + i,
&conn) == 0)
if (fdfs_server_contain1(g_tracker_group.servers + i, &conn))
{
fdfs_set_server_info_index1(g_tracker_group.servers + i, &conn);
g_tracker_group.server_index = i;
break;
}
@ -277,32 +277,35 @@ static int list_storages(FDFSGroupStat *pGroupStat)
char szSyncedDelaySeconds[128];
char szHostname[128];
char szHostnamePrompt[128+8];
char szDiskTotalSpace[32];
char szDiskFreeSpace[32];
char szTrunkSpace[32];
int k;
int max_last_source_update;
printf( "group name = %s\n" \
"disk total space = %"PRId64" MB\n" \
"disk free space = %"PRId64" MB\n" \
"trunk free space = %"PRId64" MB\n" \
"storage server count = %d\n" \
"active server count = %d\n" \
"storage server port = %d\n" \
"storage HTTP port = %d\n" \
"store path count = %d\n" \
"subdir count per path = %d\n" \
"current write server index = %d\n" \
"current trunk file id = %d\n\n", \
pGroupStat->group_name, \
pGroupStat->total_mb, \
pGroupStat->free_mb, \
pGroupStat->trunk_free_mb, \
pGroupStat->count, \
pGroupStat->active_count, \
pGroupStat->storage_port, \
pGroupStat->storage_http_port, \
pGroupStat->store_path_count, \
pGroupStat->subdir_count_per_path, \
pGroupStat->current_write_server, \
printf( "group name = %s\n"
"disk total space = %s MB\n"
"disk free space = %s MB\n"
"trunk free space = %s MB\n"
"storage server count = %d\n"
"active server count = %d\n"
"storage server port = %d\n"
"storage HTTP port = %d\n"
"store path count = %d\n"
"subdir count per path = %d\n"
"current write server index = %d\n"
"current trunk file id = %d\n\n",
pGroupStat->group_name,
long_to_comma_str(pGroupStat->total_mb, szDiskTotalSpace),
long_to_comma_str(pGroupStat->free_mb, szDiskFreeSpace),
long_to_comma_str(pGroupStat->trunk_free_mb, szTrunkSpace),
pGroupStat->count,
pGroupStat->active_count,
pGroupStat->storage_port,
pGroupStat->storage_http_port,
pGroupStat->store_path_count,
pGroupStat->subdir_count_per_path,
pGroupStat->current_write_server,
pGroupStat->current_trunk_file_id
);
@ -352,8 +355,12 @@ static int list_storages(FDFSGroupStat *pGroupStat)
int second;
char szDelayTime[64];
delay_seconds = (int)(max_last_source_update - \
delay_seconds = (int)(max_last_source_update -
pStorageStat->last_synced_timestamp);
if (delay_seconds < 0)
{
delay_seconds = 0;
}
day = delay_seconds / (24 * 3600);
remain_seconds = delay_seconds % (24 * 3600);
hour = remain_seconds / 3600;
@ -385,7 +392,8 @@ static int list_storages(FDFSGroupStat *pGroupStat)
}
}
getHostnameByIp(pStorage->ip_addr, szHostname, sizeof(szHostname));
//getHostnameByIp(pStorage->ip_addr, szHostname, sizeof(szHostname));
*szHostname = '\0';
if (*szHostname != '\0')
{
sprintf(szHostnamePrompt, " (%s)", szHostname);
@ -406,138 +414,138 @@ static int list_storages(FDFSGroupStat *pGroupStat)
*szUpTime = '\0';
}
printf( "\tStorage %d:\n" \
"\t\tid = %s\n" \
"\t\tip_addr = %s%s %s\n" \
"\t\thttp domain = %s\n" \
"\t\tversion = %s\n" \
"\t\tjoin time = %s\n" \
"\t\tup time = %s\n" \
"\t\ttotal storage = %d MB\n" \
"\t\tfree storage = %d MB\n" \
"\t\tupload priority = %d\n" \
"\t\tstore_path_count = %d\n" \
"\t\tsubdir_count_per_path = %d\n" \
"\t\tstorage_port = %d\n" \
"\t\tstorage_http_port = %d\n" \
"\t\tcurrent_write_path = %d\n" \
"\t\tsource storage id = %s\n" \
"\t\tif_trunk_server = %d\n" \
"\t\tconnection.alloc_count = %d\n" \
"\t\tconnection.current_count = %d\n" \
"\t\tconnection.max_count = %d\n" \
"\t\ttotal_upload_count = %"PRId64"\n" \
"\t\tsuccess_upload_count = %"PRId64"\n" \
"\t\ttotal_append_count = %"PRId64"\n" \
"\t\tsuccess_append_count = %"PRId64"\n" \
"\t\ttotal_modify_count = %"PRId64"\n" \
"\t\tsuccess_modify_count = %"PRId64"\n" \
"\t\ttotal_truncate_count = %"PRId64"\n" \
"\t\tsuccess_truncate_count = %"PRId64"\n" \
"\t\ttotal_set_meta_count = %"PRId64"\n" \
"\t\tsuccess_set_meta_count = %"PRId64"\n" \
"\t\ttotal_delete_count = %"PRId64"\n" \
"\t\tsuccess_delete_count = %"PRId64"\n" \
"\t\ttotal_download_count = %"PRId64"\n" \
"\t\tsuccess_download_count = %"PRId64"\n" \
"\t\ttotal_get_meta_count = %"PRId64"\n" \
"\t\tsuccess_get_meta_count = %"PRId64"\n" \
"\t\ttotal_create_link_count = %"PRId64"\n" \
"\t\tsuccess_create_link_count = %"PRId64"\n"\
"\t\ttotal_delete_link_count = %"PRId64"\n" \
"\t\tsuccess_delete_link_count = %"PRId64"\n" \
"\t\ttotal_upload_bytes = %"PRId64"\n" \
"\t\tsuccess_upload_bytes = %"PRId64"\n" \
"\t\ttotal_append_bytes = %"PRId64"\n" \
"\t\tsuccess_append_bytes = %"PRId64"\n" \
"\t\ttotal_modify_bytes = %"PRId64"\n" \
"\t\tsuccess_modify_bytes = %"PRId64"\n" \
"\t\tstotal_download_bytes = %"PRId64"\n" \
"\t\tsuccess_download_bytes = %"PRId64"\n" \
"\t\ttotal_sync_in_bytes = %"PRId64"\n" \
"\t\tsuccess_sync_in_bytes = %"PRId64"\n" \
"\t\ttotal_sync_out_bytes = %"PRId64"\n" \
"\t\tsuccess_sync_out_bytes = %"PRId64"\n" \
"\t\ttotal_file_open_count = %"PRId64"\n" \
"\t\tsuccess_file_open_count = %"PRId64"\n" \
"\t\ttotal_file_read_count = %"PRId64"\n" \
"\t\tsuccess_file_read_count = %"PRId64"\n" \
"\t\ttotal_file_write_count = %"PRId64"\n" \
"\t\tsuccess_file_write_count = %"PRId64"\n" \
"\t\tlast_heart_beat_time = %s\n" \
"\t\tlast_source_update = %s\n" \
"\t\tlast_sync_update = %s\n" \
"\t\tlast_synced_timestamp = %s %s\n", \
++k, pStorage->id, pStorage->ip_addr, \
szHostnamePrompt, get_storage_status_caption( \
pStorage->status), pStorage->domain_name, \
pStorage->version, \
formatDatetime(pStorage->join_time, \
"%Y-%m-%d %H:%M:%S", \
szJoinTime, sizeof(szJoinTime)), \
szUpTime, pStorage->total_mb, \
pStorage->free_mb, \
pStorage->upload_priority, \
pStorage->store_path_count, \
pStorage->subdir_count_per_path, \
pStorage->storage_port, \
pStorage->storage_http_port, \
pStorage->current_write_path, \
pStorage->src_id, \
pStorage->if_trunk_server, \
pStorageStat->connection.alloc_count, \
pStorageStat->connection.current_count, \
pStorageStat->connection.max_count, \
pStorageStat->total_upload_count, \
pStorageStat->success_upload_count, \
pStorageStat->total_append_count, \
pStorageStat->success_append_count, \
pStorageStat->total_modify_count, \
pStorageStat->success_modify_count, \
pStorageStat->total_truncate_count, \
pStorageStat->success_truncate_count, \
pStorageStat->total_set_meta_count, \
pStorageStat->success_set_meta_count, \
pStorageStat->total_delete_count, \
pStorageStat->success_delete_count, \
pStorageStat->total_download_count, \
pStorageStat->success_download_count, \
pStorageStat->total_get_meta_count, \
pStorageStat->success_get_meta_count, \
pStorageStat->total_create_link_count, \
pStorageStat->success_create_link_count, \
pStorageStat->total_delete_link_count, \
pStorageStat->success_delete_link_count, \
pStorageStat->total_upload_bytes, \
pStorageStat->success_upload_bytes, \
pStorageStat->total_append_bytes, \
pStorageStat->success_append_bytes, \
pStorageStat->total_modify_bytes, \
pStorageStat->success_modify_bytes, \
pStorageStat->total_download_bytes, \
pStorageStat->success_download_bytes, \
pStorageStat->total_sync_in_bytes, \
pStorageStat->success_sync_in_bytes, \
pStorageStat->total_sync_out_bytes, \
pStorageStat->success_sync_out_bytes, \
pStorageStat->total_file_open_count, \
pStorageStat->success_file_open_count, \
pStorageStat->total_file_read_count, \
pStorageStat->success_file_read_count, \
pStorageStat->total_file_write_count, \
pStorageStat->success_file_write_count, \
formatDatetime(pStorageStat->last_heart_beat_time, \
"%Y-%m-%d %H:%M:%S", \
szLastHeartBeatTime, sizeof(szLastHeartBeatTime)), \
formatDatetime(pStorageStat->last_source_update, \
"%Y-%m-%d %H:%M:%S", \
szSrcUpdTime, sizeof(szSrcUpdTime)), \
formatDatetime(pStorageStat->last_sync_update, \
"%Y-%m-%d %H:%M:%S", \
szSyncUpdTime, sizeof(szSyncUpdTime)), \
formatDatetime(pStorageStat->last_synced_timestamp, \
"%Y-%m-%d %H:%M:%S", \
szSyncedTimestamp, sizeof(szSyncedTimestamp)),\
printf( "\tStorage %d:\n"
"\t\tid = %s\n"
"\t\tip_addr = %s%s %s\n"
"\t\thttp domain = %s\n"
"\t\tversion = %s\n"
"\t\tjoin time = %s\n"
"\t\tup time = %s\n"
"\t\ttotal storage = %s MB\n"
"\t\tfree storage = %s MB\n"
"\t\tupload priority = %d\n"
"\t\tstore_path_count = %d\n"
"\t\tsubdir_count_per_path = %d\n"
"\t\tstorage_port = %d\n"
"\t\tstorage_http_port = %d\n"
"\t\tcurrent_write_path = %d\n"
"\t\tsource storage id = %s\n"
"\t\tif_trunk_server = %d\n"
"\t\tconnection.alloc_count = %d\n"
"\t\tconnection.current_count = %d\n"
"\t\tconnection.max_count = %d\n"
"\t\ttotal_upload_count = %"PRId64"\n"
"\t\tsuccess_upload_count = %"PRId64"\n"
"\t\ttotal_append_count = %"PRId64"\n"
"\t\tsuccess_append_count = %"PRId64"\n"
"\t\ttotal_modify_count = %"PRId64"\n"
"\t\tsuccess_modify_count = %"PRId64"\n"
"\t\ttotal_truncate_count = %"PRId64"\n"
"\t\tsuccess_truncate_count = %"PRId64"\n"
"\t\ttotal_set_meta_count = %"PRId64"\n"
"\t\tsuccess_set_meta_count = %"PRId64"\n"
"\t\ttotal_delete_count = %"PRId64"\n"
"\t\tsuccess_delete_count = %"PRId64"\n"
"\t\ttotal_download_count = %"PRId64"\n"
"\t\tsuccess_download_count = %"PRId64"\n"
"\t\ttotal_get_meta_count = %"PRId64"\n"
"\t\tsuccess_get_meta_count = %"PRId64"\n"
"\t\ttotal_create_link_count = %"PRId64"\n"
"\t\tsuccess_create_link_count = %"PRId64"\n"
"\t\ttotal_delete_link_count = %"PRId64"\n"
"\t\tsuccess_delete_link_count = %"PRId64"\n"
"\t\ttotal_upload_bytes = %"PRId64"\n"
"\t\tsuccess_upload_bytes = %"PRId64"\n"
"\t\ttotal_append_bytes = %"PRId64"\n"
"\t\tsuccess_append_bytes = %"PRId64"\n"
"\t\ttotal_modify_bytes = %"PRId64"\n"
"\t\tsuccess_modify_bytes = %"PRId64"\n"
"\t\tstotal_download_bytes = %"PRId64"\n"
"\t\tsuccess_download_bytes = %"PRId64"\n"
"\t\ttotal_sync_in_bytes = %"PRId64"\n"
"\t\tsuccess_sync_in_bytes = %"PRId64"\n"
"\t\ttotal_sync_out_bytes = %"PRId64"\n"
"\t\tsuccess_sync_out_bytes = %"PRId64"\n"
"\t\ttotal_file_open_count = %"PRId64"\n"
"\t\tsuccess_file_open_count = %"PRId64"\n"
"\t\ttotal_file_read_count = %"PRId64"\n"
"\t\tsuccess_file_read_count = %"PRId64"\n"
"\t\ttotal_file_write_count = %"PRId64"\n"
"\t\tsuccess_file_write_count = %"PRId64"\n"
"\t\tlast_heart_beat_time = %s\n"
"\t\tlast_source_update = %s\n"
"\t\tlast_sync_update = %s\n"
"\t\tlast_synced_timestamp = %s %s\n",
++k, pStorage->id, pStorage->ip_addr,
szHostnamePrompt, get_storage_status_caption(
pStorage->status), pStorage->domain_name,
pStorage->version,
formatDatetime(pStorage->join_time,
"%Y-%m-%d %H:%M:%S",
szJoinTime, sizeof(szJoinTime)), szUpTime,
long_to_comma_str(pStorage->total_mb, szDiskTotalSpace),
long_to_comma_str(pStorage->free_mb, szDiskFreeSpace),
pStorage->upload_priority,
pStorage->store_path_count,
pStorage->subdir_count_per_path,
pStorage->storage_port,
pStorage->storage_http_port,
pStorage->current_write_path,
pStorage->src_id,
pStorage->if_trunk_server,
pStorageStat->connection.alloc_count,
pStorageStat->connection.current_count,
pStorageStat->connection.max_count,
pStorageStat->total_upload_count,
pStorageStat->success_upload_count,
pStorageStat->total_append_count,
pStorageStat->success_append_count,
pStorageStat->total_modify_count,
pStorageStat->success_modify_count,
pStorageStat->total_truncate_count,
pStorageStat->success_truncate_count,
pStorageStat->total_set_meta_count,
pStorageStat->success_set_meta_count,
pStorageStat->total_delete_count,
pStorageStat->success_delete_count,
pStorageStat->total_download_count,
pStorageStat->success_download_count,
pStorageStat->total_get_meta_count,
pStorageStat->success_get_meta_count,
pStorageStat->total_create_link_count,
pStorageStat->success_create_link_count,
pStorageStat->total_delete_link_count,
pStorageStat->success_delete_link_count,
pStorageStat->total_upload_bytes,
pStorageStat->success_upload_bytes,
pStorageStat->total_append_bytes,
pStorageStat->success_append_bytes,
pStorageStat->total_modify_bytes,
pStorageStat->success_modify_bytes,
pStorageStat->total_download_bytes,
pStorageStat->success_download_bytes,
pStorageStat->total_sync_in_bytes,
pStorageStat->success_sync_in_bytes,
pStorageStat->total_sync_out_bytes,
pStorageStat->success_sync_out_bytes,
pStorageStat->total_file_open_count,
pStorageStat->success_file_open_count,
pStorageStat->total_file_read_count,
pStorageStat->success_file_read_count,
pStorageStat->total_file_write_count,
pStorageStat->success_file_write_count,
formatDatetime(pStorageStat->last_heart_beat_time,
"%Y-%m-%d %H:%M:%S",
szLastHeartBeatTime, sizeof(szLastHeartBeatTime)),
formatDatetime(pStorageStat->last_source_update,
"%Y-%m-%d %H:%M:%S",
szSrcUpdTime, sizeof(szSrcUpdTime)),
formatDatetime(pStorageStat->last_sync_update,
"%Y-%m-%d %H:%M:%S",
szSyncUpdTime, sizeof(szSyncUpdTime)),
formatDatetime(pStorageStat->last_synced_timestamp,
"%Y-%m-%d %H:%M:%S",
szSyncedTimestamp, sizeof(szSyncedTimestamp)),
szSyncedDelaySeconds);
}

68
client/fdfs_regenerate_filename.c Normal file
View File

@ -0,0 +1,68 @@
/**
* Copyright (C) 2008 Happy Fish / YuQing
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "fdfs_client.h"
#include "fastcommon/logger.h"
int main(int argc, char *argv[])
{
char *conf_filename;
ConnectionInfo *pTrackerServer;
int result;
char appender_file_id[128];
char new_file_id[128];
if (argc < 3)
{
fprintf(stderr, "regenerate filename for the appender file.\n"
"NOTE: the regenerated file will be a normal file!\n"
"Usage: %s <config_file> <appender_file_id>\n",
argv[0]);
return 1;
}
log_init();
g_log_context.log_level = LOG_ERR;
conf_filename = argv[1];
if ((result=fdfs_client_init(conf_filename)) != 0)
{
return result;
}
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
{
fdfs_client_destroy();
return errno != 0 ? errno : ECONNREFUSED;
}
snprintf(appender_file_id, sizeof(appender_file_id), "%s", argv[2]);
if ((result=storage_regenerate_appender_filename1(pTrackerServer,
NULL, appender_file_id, new_file_id)) != 0)
{
fprintf(stderr, "regenerate file %s fail, "
"error no: %d, error info: %s\n",
appender_file_id, result, STRERROR(result));
return result;
}
printf("%s\n", new_file_id);
tracker_close_connection_ex(pTrackerServer, true);
fdfs_client_destroy();
return result;
}

4
client/fdfs_test.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -84,7 +84,7 @@ int main(int argc, char *argv[])
"\nCopyright (C) 2008, Happy Fish / YuQing\n" \
"\nFastDFS may be copied only under the terms of the GNU General\n" \
"Public License V3, which may be found in the FastDFS source kit.\n" \
"Please visit the FastDFS Home Page http://www.csource.org/ \n" \
"Please visit the FastDFS Home Page http://www.fastken.com/ \n" \
"for more detail.\n\n" \
, g_fdfs_version.major, g_fdfs_version.minor);

4
client/fdfs_test1.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -82,7 +82,7 @@ int main(int argc, char *argv[])
"\nCopyright (C) 2008, Happy Fish / YuQing\n" \
"\nFastDFS may be copied only under the terms of the GNU General\n" \
"Public License V3, which may be found in the FastDFS source kit.\n" \
"Please visit the FastDFS Home Page http://www.csource.org/ \n" \
"Please visit the FastDFS Home Page http://www.fastken.com/ \n" \
"for more detail.\n\n" \
, g_fdfs_version.major, g_fdfs_version.minor);

2
client/fdfs_upload_appender.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>

2
client/fdfs_upload_file.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>

156
client/storage_client.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
@ -35,20 +35,20 @@ static struct base64_context the_base64_context;
static int the_base64_context_inited = 0;
#define FDFS_SPLIT_GROUP_NAME_AND_FILENAME(file_id) \
char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; \
char in_file_id[FDFS_GROUP_NAME_MAX_LEN + 128]; \
char *group_name; \
char *filename; \
char *pSeperator; \
\
snprintf(new_file_id, sizeof(new_file_id), "%s", file_id); \
pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR); \
snprintf(in_file_id, sizeof(in_file_id), "%s", file_id); \
pSeperator = strchr(in_file_id, FDFS_FILE_ID_SEPERATOR); \
if (pSeperator == NULL) \
{ \
return EINVAL; \
} \
\
*pSeperator = '\0'; \
group_name = new_file_id; \
group_name = in_file_id; \
filename = pSeperator + 1; \
#define storage_get_read_connection(pTrackerServer, \
@ -794,22 +794,16 @@ int storage_do_upload_file1(ConnectionInfo *pTrackerServer, \
STORAGE_PROTO_CMD_UPLOAD_FILE and
STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE:
1 byte: store path index
8 bytes: meta data bytes
8 bytes: file size
FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name
meta data bytes: each meta data seperated by \x01,
name and value seperated by \x02
file size bytes: file content
STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE:
8 bytes: master filename length
8 bytes: meta data bytes
8 bytes: file size
FDFS_FILE_PREFIX_MAX_LEN bytes : filename prefix
FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name, do not include dot (.)
master filename bytes: master filename
meta data bytes: each meta data seperated by \x01,
name and value seperated by \x02
file size bytes: file content
**/
int storage_do_upload_file(ConnectionInfo *pTrackerServer, \
@ -1808,7 +1802,7 @@ int storage_do_append_file(ConnectionInfo *pTrackerServer, \
}
/**
STORAGE_PROTO_CMD_APPEND_FILE:
STORAGE_PROTO_CMD_MODIFY_FILE:
8 bytes: appender filename length
8 bytes: file offset
8 bytes: file size
@ -2181,7 +2175,7 @@ int fdfs_get_file_info_ex(const char *group_name, const char *remote_filename, \
if (pStorageId != NULL)
{
strcpy(pFileInfo->source_ip_addr,
pStorageId->ip_addrs.ips[0]);
pStorageId->ip_addrs.ips[0].address);
}
else
{
@ -2203,8 +2197,21 @@ int fdfs_get_file_info_ex(const char *group_name, const char *remote_filename, \
pFileInfo->create_timestamp = buff2int(buff + sizeof(int));
pFileInfo->file_size = buff2long(buff + sizeof(int) * 2);
if (IS_SLAVE_FILE(filename_len, pFileInfo->file_size) || \
IS_APPENDER_FILE(pFileInfo->file_size) || \
if (IS_APPENDER_FILE(pFileInfo->file_size))
{
pFileInfo->file_type = FDFS_FILE_TYPE_APPENDER;
}
else if (IS_SLAVE_FILE(filename_len, pFileInfo->file_size))
{
pFileInfo->file_type = FDFS_FILE_TYPE_SLAVE;
}
else
{
pFileInfo->file_type = FDFS_FILE_TYPE_NORMAL;
}
if (pFileInfo->file_type == FDFS_FILE_TYPE_SLAVE ||
pFileInfo->file_type == FDFS_FILE_TYPE_APPENDER ||
(*(pFileInfo->source_ip_addr) == '\0' && get_from_server))
{ //slave file or appender file
if (get_from_server)
@ -2218,21 +2225,24 @@ int fdfs_get_file_info_ex(const char *group_name, const char *remote_filename, \
return result;
}
result = storage_query_file_info(conn, \
result = storage_query_file_info(conn,
NULL, group_name, remote_filename, pFileInfo);
tracker_close_connection_ex(conn, result != 0 && \
tracker_close_connection_ex(conn, result != 0 &&
result != ENOENT);
pFileInfo->get_from_server = true;
return result;
}
else
{
pFileInfo->get_from_server = false;
pFileInfo->file_size = -1;
return 0;
}
}
else //master file (normal file)
{
pFileInfo->get_from_server = false;
if ((pFileInfo->file_size >> 63) != 0)
{
pFileInfo->file_size &= 0xFFFFFFFF; //low 32 bits is file size
@ -2352,3 +2362,115 @@ int storage_truncate_file(ConnectionInfo *pTrackerServer, \
return result;
}
int storage_regenerate_appender_filename(ConnectionInfo *pTrackerServer,
ConnectionInfo *pStorageServer, const char *group_name,
const char *appender_filename, char *new_group_name,
char *new_remote_filename)
{
TrackerHeader *pHeader;
int result;
char out_buff[512];
char in_buff[256];
char *p;
char *pInBuff;
int64_t in_bytes;
ConnectionInfo storageServer;
bool new_connection;
int appender_filename_len;
appender_filename_len = strlen(appender_filename);
if ((result=storage_get_update_connection(pTrackerServer,
&pStorageServer, group_name, appender_filename,
&storageServer, &new_connection)) != 0)
{
return result;
}
do
{
pHeader = (TrackerHeader *)out_buff;
p = out_buff + sizeof(TrackerHeader);
memcpy(p, appender_filename, appender_filename_len);
p += appender_filename_len;
long2buff((p - out_buff) - sizeof(TrackerHeader),
pHeader->pkg_len);
pHeader->cmd = STORAGE_PROTO_CMD_REGENERATE_APPENDER_FILENAME;
pHeader->status = 0;
if ((result=tcpsenddata_nb(pStorageServer->sock, out_buff,
p - out_buff, g_fdfs_network_timeout)) != 0)
{
logError("file: "__FILE__", line: %d, "
"send data to storage server %s:%d fail, "
"errno: %d, error info: %s", __LINE__,
pStorageServer->ip_addr, pStorageServer->port,
result, STRERROR(result));
break;
}
pInBuff = in_buff;
if ((result=fdfs_recv_response(pStorageServer,
&pInBuff, sizeof(in_buff), &in_bytes)) != 0)
{
logError("file: "__FILE__", line: %d, "
"fdfs_recv_response fail, result: %d",
__LINE__, result);
break;
}
if (in_bytes <= FDFS_GROUP_NAME_MAX_LEN)
{
logError("file: "__FILE__", line: %d, "
"storage server %s:%d response data "
"length: %"PRId64" is invalid, "
"should > %d", __LINE__,
pStorageServer->ip_addr, pStorageServer->port,
in_bytes, FDFS_GROUP_NAME_MAX_LEN);
result = EINVAL;
break;
}
in_buff[in_bytes] = '\0';
memcpy(new_group_name, in_buff, FDFS_GROUP_NAME_MAX_LEN);
new_group_name[FDFS_GROUP_NAME_MAX_LEN] = '\0';
memcpy(new_remote_filename, in_buff + FDFS_GROUP_NAME_MAX_LEN,
in_bytes - FDFS_GROUP_NAME_MAX_LEN + 1);
} while (0);
if (new_connection)
{
tracker_close_connection_ex(pStorageServer, result != 0);
}
return result;
}
int storage_regenerate_appender_filename1(ConnectionInfo *pTrackerServer,
ConnectionInfo *pStorageServer, const char *appender_file_id,
char *new_file_id)
{
int result;
char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1];
char new_remote_filename[128];
FDFS_SPLIT_GROUP_NAME_AND_FILENAME(appender_file_id)
result = storage_regenerate_appender_filename(pTrackerServer,
pStorageServer, group_name, filename,
new_group_name, new_remote_filename);
if (result == 0)
{
sprintf(new_file_id, "%s%c%s", new_group_name,
FDFS_FILE_ID_SEPERATOR, new_remote_filename);
}
else
{
new_file_id[0] = '\0';
}
return result;
}

19
client/storage_client.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#ifndef STORAGE_CLIENT_H
@ -563,6 +563,23 @@ int fdfs_get_file_info_ex(const char *group_name, const char *remote_filename, \
const bool get_from_server, FDFSFileInfo *pFileInfo);
/**
* regenerate normal filename for appender file
* Note: the appender file will change to normal file
* params:
* pTrackerServer: the tracker server
* pStorageServer: the storage server
* group_name: the group name
* appender_filename: the appender filename
* new_group_name: return the new group name
* new_remote_filename: return the new filename
* return: 0 success, !=0 fail, return the error code
**/
int storage_regenerate_appender_filename(ConnectionInfo *pTrackerServer,
ConnectionInfo *pStorageServer, const char *group_name,
const char *appender_filename, char *new_group_name,
char *new_remote_filename);
#ifdef __cplusplus
}
#endif

18
client/storage_client1.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#ifndef STORAGE_CLIENT1_H
@ -524,6 +524,22 @@ int fdfs_get_file_info_ex1(const char *file_id, const bool get_from_server, \
int storage_file_exist1(ConnectionInfo *pTrackerServer, \
ConnectionInfo *pStorageServer, \
const char *file_id);
/**
* regenerate normal filename for appender file
* Note: the appender file will change to normal file
* params:
* pTrackerServer: the tracker server
* pStorageServer: the storage server
* group_name: the group name
* appender_file_id: the appender file id
* file_id: regenerated file id return by storage server
* return: 0 success, !=0 fail, return the error code
**/
int storage_regenerate_appender_filename1(ConnectionInfo *pTrackerServer,
ConnectionInfo *pStorageServer, const char *appender_file_id,
char *new_file_id);
#ifdef __cplusplus
}
#endif

3
client/test/Makefile.in
View File

@ -1,7 +1,8 @@
.SUFFIXES: .c .o
COMPILE = $(CC) $(CFLAGS)
INC_PATH = -I/usr/include/fastcommon -I/usr/include/fastdfs
INC_PATH = -I/usr/include/fastcommon -I/usr/include/fastdfs \
-I/usr/local/include/fastcommon -I/usr/local/include/fastdfs
LIB_PATH = -L/usr/local/lib -lfastcommon -lfdfsclient $(LIBS)
TARGET_PATH = $(TARGET_PATH)

35
client/test/fdfs_monitor.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -12,6 +12,7 @@
#include <string.h>
#include <errno.h>
#include <signal.h>
#include <netdb.h>
#include <sys/types.h>
#include "fastcommon/sockopt.h"
#include "fastcommon/logger.h"
@ -25,8 +26,11 @@ static int list_all_groups(const char *group_name);
static void usage(char *argv[])
{
printf("Usage: %s <config_file> [-h <tracker_server>] [list|delete|set_trunk_server <group_name> " \
"[storage_id]]\n", argv[0]);
printf("Usage: %s <config_file> [-h <tracker_server>] "
"[list|delete|set_trunk_server <group_name> [storage_id]]\n"
"\tthe tracker server format: host[:port], "
"the tracker default port is %d\n\n",
argv[0], FDFS_TRACKER_SERVER_DEF_PORT);
}
int main(int argc, char *argv[])
@ -115,21 +119,19 @@ int main(int argc, char *argv[])
else
{
int i;
char ip_addr[IP_ADDRESS_SIZE];
ConnectionInfo conn;
*ip_addr = '\0';
if (getIpaddrByName(tracker_server, ip_addr, sizeof(ip_addr)) \
== INADDR_NONE)
if ((result=conn_pool_parse_server_info(tracker_server, &conn,
FDFS_TRACKER_SERVER_DEF_PORT)) != 0)
{
printf("resolve ip address of tracker server: %s " \
"fail!\n", tracker_server);
return 2;
printf("resolve ip address of tracker server: %s "
"fail!, error info: %s\n", tracker_server, hstrerror(h_errno));
return result;
}
for (i=0; i<g_tracker_group.server_count; i++)
{
if (strcmp(g_tracker_group.servers[i].ip_addr, \
ip_addr) == 0)
if (fdfs_server_contain1(g_tracker_group.servers + i, &conn))
{
g_tracker_group.server_index = i;
break;
@ -143,7 +145,8 @@ int main(int argc, char *argv[])
}
}
printf("server_count=%d, server_index=%d\n", g_tracker_group.server_count, g_tracker_group.server_index);
printf("server_count=%d, server_index=%d\n",
g_tracker_group.server_count, g_tracker_group.server_index);
pTrackerServer = tracker_get_connection();
if (pTrackerServer == NULL)
@ -348,8 +351,12 @@ static int list_storages(FDFSGroupStat *pGroupStat)
int second;
char szDelayTime[64];
delay_seconds = (int)(max_last_source_update - \
delay_seconds = (int)(max_last_source_update -
pStorageStat->last_synced_timestamp);
if (delay_seconds < 0)
{
delay_seconds = 0;
}
day = delay_seconds / (24 * 3600);
remain_seconds = delay_seconds % (24 * 3600);
hour = remain_seconds / 3600;

8
client/test/fdfs_test.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -84,7 +84,7 @@ int main(int argc, char *argv[])
"\nCopyright (C) 2008, Happy Fish / YuQing\n" \
"\nFastDFS may be copied only under the terms of the GNU General\n" \
"Public License V3, which may be found in the FastDFS source kit.\n" \
"Please visit the FastDFS Home Page http://www.csource.org/ \n" \
"Please visit the FastDFS Home Page http://www.fastken.com/ \n" \
"for more detail.\n\n" \
, g_fdfs_version.major, g_fdfs_version.minor);
@ -194,7 +194,7 @@ int main(int argc, char *argv[])
group_name, storageServer.ip_addr, \
storageServer.port);
if ((pStorageServer=tracker_connect_server(&storageServer, \
if ((pStorageServer=tracker_make_connection(&storageServer, \
&result)) == NULL)
{
fdfs_client_destroy();
@ -491,7 +491,7 @@ int main(int argc, char *argv[])
printf("storage=%s:%d\n", storageServer.ip_addr, \
storageServer.port);
if ((pStorageServer=tracker_connect_server(&storageServer, \
if ((pStorageServer=tracker_make_connection(&storageServer, \
&result)) == NULL)
{
fdfs_client_destroy();

8
client/test/fdfs_test1.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -82,7 +82,7 @@ int main(int argc, char *argv[])
"\nCopyright (C) 2008, Happy Fish / YuQing\n" \
"\nFastDFS may be copied only under the terms of the GNU General\n" \
"Public License V3, which may be found in the FastDFS source kit.\n" \
"Please visit the FastDFS Home Page http://www.csource.org/ \n" \
"Please visit the FastDFS Home Page http://www.fastken.com/ \n" \
"for more detail.\n\n" \
, g_fdfs_version.major, g_fdfs_version.minor);
@ -191,7 +191,7 @@ int main(int argc, char *argv[])
group_name, storageServer.ip_addr, \
storageServer.port);
if ((pStorageServer=tracker_connect_server(&storageServer, \
if ((pStorageServer=tracker_make_connection(&storageServer, \
&result)) == NULL)
{
fdfs_client_destroy();
@ -458,7 +458,7 @@ int main(int argc, char *argv[])
printf("storage=%s:%d\n", storageServer.ip_addr, \
storageServer.port);
if ((pStorageServer=tracker_connect_server(&storageServer, \
if ((pStorageServer=tracker_make_connection(&storageServer, \
&result)) == NULL)
{
fdfs_client_destroy();

6
client/tracker_client.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
@ -1429,8 +1429,8 @@ int tracker_set_trunk_server(TrackerServerGroup *pTrackerGroup, \
return result;
}
int tracker_get_storage_status(ConnectionInfo *pTrackerServer, \
const char *group_name, const char *ip_addr, \
int tracker_get_storage_status(ConnectionInfo *pTrackerServer,
const char *group_name, const char *ip_addr,
FDFSStorageBrief *pDestBuff)
{
TrackerHeader *pHeader;

2
client/tracker_client.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#ifndef TRACKER_CLIENT_H

4
common/fdfs_define.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdfs_define.h
@ -14,7 +14,7 @@
#include <pthread.h>
#include "fastcommon/common_define.h"
#define FDFS_TRACKER_SERVER_DEF_PORT 22000
#define FDFS_TRACKER_SERVER_DEF_PORT 22122
#define FDFS_STORAGE_SERVER_DEF_PORT 23000
#define FDFS_DEF_STORAGE_RESERVED_MB 1024
#define TRACKER_ERROR_LOG_FILENAME "trackerd"

4
common/fdfs_global.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <time.h>
@ -23,7 +23,7 @@
int g_fdfs_connect_timeout = DEFAULT_CONNECT_TIMEOUT;
int g_fdfs_network_timeout = DEFAULT_NETWORK_TIMEOUT;
char g_fdfs_base_path[MAX_PATH_SIZE] = {'/', 't', 'm', 'p', '\0'};
Version g_fdfs_version = {5, 12};
Version g_fdfs_version = {6, 7};
bool g_use_connection_pool = false;
ConnectionPool g_connection_pool;
int g_connection_pool_max_idle_time = 3600;

2
common/fdfs_global.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdfs_global.h

2
common/fdfs_http_shared.c
View File

@ -4,7 +4,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <time.h>

2
common/fdfs_http_shared.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#ifndef _FDFS_HTTP_SHARED_H

2
common/mime_file_parser.c
View File

@ -4,7 +4,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <time.h>

2
common/mime_file_parser.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#ifndef _MINE_FILE_PARSER_H

9
conf/client.conf
View File

@ -1,6 +1,7 @@
# connect timeout in seconds
# default value is 30s
connect_timeout=10
# Note: in the intranet network (LAN), 2 seconds is enough.
connect_timeout = 5
# network timeout in seconds
# default value is 30s
@ -13,8 +14,10 @@ base_path=/home/yuqing/fastdfs
# the value format of tracker_server is "HOST:PORT",
# the HOST can be hostname or ip address,
# and the HOST can be dual IPs or hostnames seperated by comma,
# the dual IPS must be an intranet IP and an extranet IP.
# such as: 192.168.2.100,122.244.141.46
# the dual IPS must be an inner (intranet) IP and an outer (extranet) IP,
# or two different types of inner (intranet) IPs.
# for example: 192.168.2.100,122.244.141.46:22122
# another eg.: 192.168.1.10,172.17.4.21:22122
tracker_server = 192.168.0.196:22122
tracker_server = 192.168.0.197:22122

113
conf/storage.conf
View File

@ -7,7 +7,7 @@ disabled=false
#
# comment or remove this item for fetching from tracker server,
# in this case, use_storage_id must set to true in tracker.conf,
# and storage_ids.conf must be configed correctly.
# and storage_ids.conf must be configured correctly.
group_name = group1
# bind an address of this host
@ -16,7 +16,7 @@ bind_addr=
# if bind an address of this host when connect to other servers
# (this storage server as a client)
# true for binding the address configed by above parameter: "bind_addr"
# true for binding the address configured by the above parameter: "bind_addr"
# false for binding any address of this host
client_bind = true
@ -24,41 +24,49 @@ client_bind=true
port = 23000
# connect timeout in seconds
# default value is 30s
connect_timeout=10
# default value is 30
# Note: in the intranet network (LAN), 2 seconds is enough.
connect_timeout = 5
# network timeout in seconds
# default value is 30s
# network timeout in seconds for send and recv
# default value is 30
network_timeout = 60
# heart beat interval in seconds
# the heart beat interval in seconds
# the storage server send heartbeat to tracker server periodically
# default value is 30
heart_beat_interval = 30
# disk usage report interval in seconds
# the storage server send disk usage report to tracker server periodically
# default value is 300
stat_report_interval = 60
# the base path to store data and log files
# NOTE: the binlog files maybe are large, make sure
# the base path has enough disk space,
# eg. the disk free space should > 50GB
base_path = /home/yuqing/fastdfs
# max concurrent connections the server supported
# default value is 256
# more max_connections means more memory will be used
# max concurrent connections the server supported,
# you should set this parameter larger, eg. 10240
# default value is 256
max_connections = 1024
# the buff size to recv / send data
# the buff size to recv / send data from/to network
# this parameter must more than 8KB
# 256KB or 512KB is recommended
# default value is 64KB
# since V2.00
buff_size = 256KB
# accept thread count
# default value is 1
# default value is 1 which is recommended
# since V4.07
accept_threads = 1
# work thread count, should <= max_connections
# work thread deal network io
# work thread count
# work threads to deal network io
# default value is 4
# since V2.00
work_threads = 4
@ -70,13 +78,13 @@ work_threads=4
# since V2.00
disk_rw_separated = true
# disk reader thread count per store base path
# disk reader thread count per store path
# for mixed read / write, this parameter can be 0
# default value is 1
# since V2.00
disk_reader_threads = 1
# disk writer thread count per store base path
# disk writer thread count per store path
# for mixed read / write, this parameter can be 0
# default value is 1
# since V2.00
@ -102,11 +110,22 @@ sync_end_time=23:59
# default value is 500
write_mark_file_freq = 500
# path(disk or mount point) count, default value is 1
# disk recovery thread count
# default value is 1
# since V6.04
disk_recovery_threads = 3
# store path (disk or mount point) count, default value is 1
store_path_count = 1
# store_path#, based 0, if store_path0 not exists, it's value is base_path
# the paths must be exist
# store_path#, based on 0, to configure the store paths to store files
# if store_path0 not exists, it's value is base_path (NOT recommended)
# the paths must be exist.
#
# IMPORTANT NOTE:
# the store paths' order is very important, don't mess up!!!
# the base_path should be independent (different) of the store paths
store_path0 = /home/yuqing/fastdfs
#store_path1 = /home/yuqing/fastdfs2
@ -118,8 +137,10 @@ subdir_count_per_path=256
# the value format of tracker_server is "HOST:PORT",
# the HOST can be hostname or ip address,
# and the HOST can be dual IPs or hostnames seperated by comma,
# the dual IPS must be an intranet IP and an extranet IP.
# such as: 192.168.2.100,122.244.141.46
# the dual IPS must be an inner (intranet) IP and an outer (extranet) IP,
# or two different types of inner (intranet) IPs.
# for example: 192.168.2.100,122.244.141.46:22122
# another eg.: 192.168.1.10,172.17.4.21:22122
tracker_server = 192.168.209.121:22122
tracker_server = 192.168.209.122:22122
@ -158,8 +179,9 @@ allow_hosts=*
# 1: random, distributted by hash code
file_distribute_path_mode = 0
# valid when file_distribute_to_path is set to 0 (round robin),
# when the written file count reaches this number, then rotate to next path
# valid when file_distribute_to_path is set to 0 (round robin).
# when the written file count reaches this number, then rotate to next path.
# rotate to the first path (00/00) after the last path (such as FF/FF).
# default value is 100
file_distribute_rotate_count = 100
@ -171,11 +193,11 @@ fsync_after_written_bytes=0
# sync log buff to disk every interval seconds
# must > 0, default value is 10 seconds
sync_log_buff_interval=10
sync_log_buff_interval = 1
# sync binlog buff / cache to disk every interval seconds
# default value is 60 seconds
sync_binlog_buff_interval=10
sync_binlog_buff_interval = 1
# sync storage stat info to disk every interval seconds
# default value is 300 seconds
@ -239,6 +261,16 @@ rotate_access_log = false
# since V4.00
access_log_rotate_time = 00:00
# if compress the old access log by gzip
# default value is false
# since V6.04
compress_old_access_log = false
# compress the access log days before
# default value is 1
# since V6.04
compress_access_log_days_before = 7
# if rotate the error log every day
# default value is false
# since V4.02
@ -250,6 +282,16 @@ rotate_error_log = false
# since V4.02
error_log_rotate_time = 00:00
# if compress the old error log by gzip
# default value is false
# since V6.04
compress_old_error_log = false
# compress the error log days before
# default value is 1
# since V6.04
compress_error_log_days_before = 7
# rotate access log when the log file exceeds this size
# 0 means never rotates log file by log file size
# default value is 0
@ -275,7 +317,7 @@ file_sync_skip_invalid_record=false
# if use connection pool
# default value is false
# since V4.05
use_connection_pool = false
use_connection_pool = true
# connections whose the idle time exceeds this time will be closed
# unit: second
@ -283,6 +325,25 @@ use_connection_pool = false
# since V4.05
connection_pool_max_idle_time = 3600
# if compress the binlog files by gzip
# default value is false
# since V6.01
compress_binlog = true
# try to compress binlog time, time format: Hour:Minute
# Hour from 0 to 23, Minute from 0 to 59
# default value is 01:30
# since V6.01
compress_binlog_time = 01:30
# if check the mark of store path to prevent confusion
# recommend to set this parameter to true
# if two storage servers (instances) MUST use a same store path for
# some specific purposes, you should set this parameter to false
# default value is true
# since V6.03
check_store_path_mark = true
# use the ip address of this storage server if domain_name is empty,
# else this domain name will ocur in the url redirected by the tracker server
http.domain_name =

12
conf/storage_ids.conf
View File

@ -1,10 +1,16 @@
# <id> <group_name> <ip_or_hostname[:port]>
#
# id is a natural number (1, 2, 3 etc.),
# 6 bits of the id length is enough, such as 100001
#
# storage ip or hostname can be dual IPs seperated by comma,
# one is an intranet IP and another is an extranet IP.
# one is an inner (intranet) IP and another is an outer (extranet) IP,
# or two different types of inner (intranet) IPs
# for example: 192.168.2.100,122.244.141.46
# another eg.: 192.168.1.10,172.17.4.21
#
# the port is optional. if you run more than one storaged instances
# in a server, you must specified the port to distinguish different instances.
# 100001 group1 192.168.0.196
# 100002 group1 192.168.0.116
100001 group1 192.168.0.196
100002 group1 192.168.0.197

101
conf/tracker.conf
View File

@ -11,39 +11,42 @@ bind_addr=
port = 22122
# connect timeout in seconds
# default value is 30s
connect_timeout=10
# default value is 30
# Note: in the intranet network (LAN), 2 seconds is enough.
connect_timeout = 5
# network timeout in seconds
# default value is 30s
# network timeout in seconds for send and recv
# default value is 30
network_timeout = 60
# the base path to store data and log files
base_path = /home/yuqing/fastdfs
# max concurrent connections this server supported
# you should set this parameter larger, eg. 102400
# max concurrent connections this server support
# you should set this parameter larger, eg. 10240
# default value is 256
max_connections = 1024
# accept thread count
# default value is 1
# default value is 1 which is recommended
# since V4.07
accept_threads = 1
# work thread count, should <= max_connections
# work thread count
# work threads to deal network io
# default value is 4
# since V2.00
work_threads = 4
# min buff size
# the min network buff size
# default value 8KB
min_buff_size = 8KB
# max buff size
# the max network buff size
# default value 128KB
max_buff_size = 128KB
# the method of selecting group to upload files
# the method for selecting group to upload files
# 0: round robin
# 1: specify group
# 2: load balance, select the max free space group to upload file
@ -72,15 +75,14 @@ download_server=0
# reserved storage space for system or other applications.
# if the free(available) space of any stoarge server in
# a group <= reserved_storage_space,
# no file can be uploaded to this group.
# a group <= reserved_storage_space, no file can be uploaded to this group.
# bytes unit can be one of follows:
### G or g for gigabyte(GB)
### M or m for megabyte(MB)
### K or k for kilobyte(KB)
### no unit for byte(B)
### XX.XX% as ratio such as reserved_storage_space = 10%
reserved_storage_space = 10%
### XX.XX% as ratio such as: reserved_storage_space = 10%
reserved_storage_space = 20%
#standard log level as syslog, case insensitive, value list:
### emerg for emergency
@ -113,7 +115,7 @@ allow_hosts=*
# sync log buff to disk every interval seconds
# default value is 10 seconds
sync_log_buff_interval = 10
sync_log_buff_interval = 1
# check storage server alive interval seconds
check_active_interval = 120
@ -150,7 +152,24 @@ slot_min_size = 256
# store the upload file to trunk file when it's size <= this value
# default value is 16MB
# since V3.00
slot_max_size = 16MB
slot_max_size = 1MB
# the alignment size to allocate the trunk space
# default value is 0 (never align)
# since V6.05
# NOTE: the larger the alignment size, the less likely of disk
# fragmentation, but the more space is wasted.
trunk_alloc_alignment_size = 256
# if merge contiguous free spaces of trunk file
# default value is false
# since V6.05
trunk_free_space_merge = true
# if delete / reclaim the unused trunk files
# default value is false
# since V6.05
delete_unused_trunk_files = false
# the trunk file size, should >= 4MB
# default value is 64MB
@ -174,8 +193,8 @@ trunk_create_file_time_base = 02:00
trunk_create_file_interval = 86400
# the threshold to create trunk file
# when the free trunk file size less than the threshold, will create
# the trunk files
# when the free trunk file size less than the threshold,
# will create he trunk files
# default value is 0
# since V3.06
trunk_create_file_space_threshold = 20G
@ -195,19 +214,41 @@ trunk_init_check_occupying = false
trunk_init_reload_from_binlog = false
# the min interval for compressing the trunk binlog file
# unit: second
# default value is 0, 0 means never compress
# unit: second, 0 means never compress
# FastDFS compress the trunk binlog when trunk init and trunk destroy
# recommand to set this parameter to 86400 (one day)
# default value is 0
# since V5.01
trunk_compress_binlog_min_interval = 0
trunk_compress_binlog_min_interval = 86400
# if use storage ID instead of IP address
# the interval for compressing the trunk binlog file
# unit: second, 0 means never compress
# recommand to set this parameter to 86400 (one day)
# default value is 0
# since V6.05
trunk_compress_binlog_interval = 86400
# compress the trunk binlog time base, time format: Hour:Minute
# Hour from 0 to 23, Minute from 0 to 59
# default value is 03:00
# since V6.05
trunk_compress_binlog_time_base = 03:00
# max backups for the trunk binlog file
# default value is 0 (never backup)
# since V6.05
trunk_binlog_max_backups = 7
# if use storage server ID instead of IP address
# if you want to use dual IPs for storage server, you MUST set
# this parameter to true, and configure the dual IPs in the file
# configured by following item "storage_ids_filename", such as storage_ids.conf
# default value is false
# since V4.00
use_storage_id = false
# specify storage ids filename, can use relative or absolute path
# this parameter is valid only when use_storage_id set to true
# since V4.00
storage_ids_filename = storage_ids.conf
@ -217,7 +258,7 @@ storage_ids_filename = storage_ids.conf
# this paramter is valid only when use_storage_id set to true
# default value is ip
# since V4.03
id_type_in_filename = ip
id_type_in_filename = id
# if store slave file use symbol link
# default value is false
@ -235,6 +276,16 @@ rotate_error_log = false
# since V4.02
error_log_rotate_time = 00:00
# if compress the old error log by gzip
# default value is false
# since V6.04
compress_old_error_log = false
# compress the error log days before
# default value is 1
# since V6.04
compress_error_log_days_before = 7
# rotate error log when the log file exceeds this size
# 0 means never rotates log file by log file size
# default value is 0
@ -249,7 +300,7 @@ log_file_keep_days = 0
# if use connection pool
# default value is false
# since V4.05
use_connection_pool = false
use_connection_pool = true
# connections whose the idle time exceeds this time will be closed
# unit: second

2
docker/dockerfile_local/Dockerfile
View File

@ -1,5 +1,5 @@
# centos 7
FROM centos
FROM centos:7
# 添加配置文件
# add profiles
ADD conf/client.conf /etc/fdfs/

2
docker/dockerfile_network/Dockerfile
View File

@ -1,5 +1,5 @@
# centos 7
FROM centos
FROM centos:7
# 添加配置文件
ADD conf/client.conf /etc/fdfs/
ADD conf/http.conf /etc/fdfs/

6
fastdfs.spec
View File

@ -3,7 +3,7 @@
%define FDFSClient libfdfsclient
%define FDFSClientDevel libfdfsclient-devel
%define FDFSTool fastdfs-tool
%define FDFSVersion 6.0.0
%define FDFSVersion 6.0.6
%define CommitVersion %(echo $COMMIT_VERSION)
Name: %{FastDFS}
@ -18,14 +18,14 @@ Source: http://perso.orange.fr/sebastien.godard/%{name}-%{version}.tar.gz
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
Requires: %__cp %__mv %__chmod %__grep %__mkdir %__install %__id
BuildRequires: libfastcommon-devel >= 1.0.41
BuildRequires: libfastcommon-devel >= 1.0.43
%description
This package provides tracker & storage of fastdfs
commit version: %{CommitVersion}
%package -n %{FDFSServer}
Requires: libfastcommon >= 1.0.41
Requires: libfastcommon >= 1.0.43
Summary: fastdfs tracker & storage
%package -n %{FDFSTool}

BIN
images/architect.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 384 KiB

13
make.sh
View File

@ -1,7 +1,6 @@
ENABLE_STATIC_LIB=0
ENABLE_SHARED_LIB=1
DESTDIR=/usr
TARGET_PREFIX=$DESTDIR
TARGET_PREFIX=$DESTDIR/usr
TARGET_CONF_PATH=$DESTDIR/etc/fdfs
TARGET_INIT_PATH=$DESTDIR/etc/init.d
@ -24,15 +23,21 @@ else
OS_BITS=64
fi
uname=$(uname)
if [ "$OS_BITS" -eq 64 ]; then
if [ "$uname" = "Darwin" ]; then
LIB_VERSION=lib
else
LIB_VERSION=lib64
fi
else
LIB_VERSION=lib
fi
LIBS=''
uname=$(uname)
if [ "$uname" = "Linux" ]; then
if [ "$OS_BITS" -eq 64 ]; then
LIBS="$LIBS -L/usr/lib64"
@ -45,6 +50,7 @@ elif [ "$uname" = "FreeBSD" ] || [ "$uname" = "Darwin" ]; then
CFLAGS="$CFLAGS"
if [ "$uname" = "Darwin" ]; then
CFLAGS="$CFLAGS -DDARWIN"
TARGET_PREFIX=$TARGET_PREFIX/local
fi
elif [ "$uname" = "SunOS" ]; then
LIBS="$LIBS -L/usr/lib"
@ -132,6 +138,7 @@ cp Makefile.in Makefile
perl -pi -e "s#\\\$\(CFLAGS\)#$CFLAGS#g" Makefile
perl -pi -e "s#\\\$\(LIBS\)#$LIBS#g" Makefile
perl -pi -e "s#\\\$\(TARGET_PREFIX\)#$TARGET_PREFIX#g" Makefile
perl -pi -e "s#\\\$\(LIB_VERSION\)#$LIB_VERSION#g" Makefile
perl -pi -e "s#\\\$\(TARGET_CONF_PATH\)#$TARGET_CONF_PATH#g" Makefile
perl -pi -e "s#\\\$\(ENABLE_STATIC_LIB\)#$ENABLE_STATIC_LIB#g" Makefile
perl -pi -e "s#\\\$\(ENABLE_SHARED_LIB\)#$ENABLE_SHARED_LIB#g" Makefile

33
php_client/fastdfs_appender_test.php
View File

@ -30,7 +30,6 @@
echo "fastdfs_storage_modify_by_filename fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n";
exit;
}
var_dump(fastdfs_get_file_info($group_name, $appender_filename));
if (!fastdfs_storage_truncate_file($group_name, $appender_filename, 0))
@ -38,9 +37,23 @@
echo "fastdfs_storage_truncate_file fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n";
exit;
}
var_dump(fastdfs_get_file_info($group_name, $appender_filename));
$new_file_info = fastdfs_storage_regenerate_appender_filename($group_name, $appender_filename);
if (!$new_file_info)
{
echo "fastdfs_storage_regenerate_appender_filename fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n";
exit;
}
$group_name = $new_file_info['group_name'];
$appender_filename = $new_file_info['filename'];
echo "regenerated file id: $group_name/$appender_filename\n";
var_dump(fastdfs_get_file_info($group_name, $appender_filename));
$result = fastdfs_storage_delete_file($group_name, $appender_filename);
echo "delete file $group_name/$appender_filename return: $result\n";
echo "function test done\n\n";
$fdfs = new FastDFS();
@ -81,8 +94,22 @@
echo "$fdfs->storage_truncate_file fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n";
exit;
}
var_dump($fdfs->get_file_info($group_name, $appender_filename));
$new_file_info = $fdfs->storage_regenerate_appender_filename($group_name, $appender_filename);
if (!$new_file_info)
{
echo "$fdfs->storage_regenerate_appender_filename fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n";
exit;
}
$group_name = $new_file_info['group_name'];
$appender_filename = $new_file_info['filename'];
echo "regenerated file id: $group_name/$appender_filename\n";
var_dump($fdfs->get_file_info($group_name, $appender_filename));
$result = $fdfs->storage_delete_file($group_name, $appender_filename);
echo "delete file $group_name/$appender_filename return: $result\n";
echo 'tracker_close_all_connections result: ' . $fdfs->tracker_close_all_connections() . "\n";
?>

28
php_client/fastdfs_appender_test1.php
View File

@ -25,7 +25,6 @@
echo "fastdfs_storage_modify_by_filename1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n";
exit;
}
var_dump(fastdfs_get_file_info1($appender_file_id));
if (!fastdfs_storage_truncate_file1($appender_file_id, 0))
@ -33,8 +32,20 @@
echo "fastdfs_storage_truncate_file1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n";
exit;
}
var_dump(fastdfs_get_file_info1($appender_file_id));
$new_file_id = fastdfs_storage_regenerate_appender_filename1($appender_file_id);
if (!$new_file_id)
{
echo "fastdfs_storage_regenerate_appender_filename1 fail, errno: " . fastdfs_get_last_error_no() . ", error info: " . fastdfs_get_last_error_info() . "\n";
exit;
}
$appender_file_id = $new_file_id;
var_dump(fastdfs_get_file_info1($appender_file_id));
$result = fastdfs_storage_delete_file1($appender_file_id);
echo "delete file $appender_file_id return: $result\n";
echo "function test done\n\n";
$fdfs = new FastDFS();
@ -69,8 +80,19 @@
echo "\$fdfs->torage_truncate_file1 torage_modify_by_filename1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n";
exit;
}
var_dump($fdfs->get_file_info1($appender_file_id));
$new_file_id = $fdfs->storage_regenerate_appender_filename1($appender_file_id);
if (!$new_file_id)
{
echo "$fdfs->storage_regenerate_appender_filename1 fail, errno: " . $fdfs->get_last_error_no() . ", error info: " . $fdfs->get_last_error_info() . "\n";
exit;
}
$appender_file_id = $new_file_id;
var_dump($fdfs->get_file_info1($appender_file_id));
$result = $fdfs->storage_delete_file1($appender_file_id);
echo "delete file $appender_file_id return: $result\n";
echo 'tracker_close_all_connections result: ' . $fdfs->tracker_close_all_connections() . "\n";
?>

289
php_client/fastdfs_client.c
View File

@ -169,6 +169,8 @@ const zend_fcall_info empty_fcall_info = { 0, NULL, NULL, NULL, NULL, 0, NULL, N
ZEND_FE(fastdfs_storage_file_exist1, NULL)
ZEND_FE(fastdfs_gen_slave_filename, NULL)
ZEND_FE(fastdfs_send_data, NULL)
ZEND_FE(fastdfs_storage_regenerate_appender_filename, NULL)
ZEND_FE(fastdfs_storage_regenerate_appender_filename1, NULL)
{NULL, NULL, NULL} /* Must be the last line */
};
@ -4047,6 +4049,195 @@ static void php_fdfs_storage_modify_file_impl( \
}
}
/*
boolean fastdfs_storage_regenerate_appender_filename(string group_name,
string appender_filename, [array tracker_server, array storage_server])
return assoc array for success, false for error
*/
static void php_fdfs_storage_regenerate_appender_filename_impl(
INTERNAL_FUNCTION_PARAMETERS, FDFSPhpContext *pContext,
const bool bFileId)
{
int result;
int argc;
char *appender_filename;
zval *tracker_obj;
zval *storage_obj;
char *group_name;
HashTable *tracker_hash;
HashTable *storage_hash;
ConnectionInfo tracker_server;
ConnectionInfo storage_server;
ConnectionInfo *pTrackerServer;
ConnectionInfo *pStorageServer;
char new_group_name[FDFS_GROUP_NAME_MAX_LEN + 1];
char new_remote_filename[128];
char new_file_id[FDFS_GROUP_NAME_MAX_LEN + 128];
zend_size_t group_name_len;
zend_size_t appender_filename_len;
int saved_tracker_sock;
int saved_storage_sock;
int min_param_count;
int max_param_count;
if (bFileId)
{
min_param_count = 1;
max_param_count = 3;
}
else
{
min_param_count = 2;
max_param_count = 4;
}
argc = ZEND_NUM_ARGS();
if (argc < min_param_count || argc > max_param_count)
{
logError("file: "__FILE__", line: %d, " \
"storage_modify_file parameters " \
"count: %d < %d or > %d", __LINE__, argc, \
min_param_count, max_param_count);
pContext->err_no = EINVAL;
RETURN_BOOL(false);
}
tracker_obj = NULL;
storage_obj = NULL;
if (bFileId)
{
char *pSeperator;
char *appender_file_id;
zend_size_t appender_file_id_len;
result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"s|aa", &appender_file_id, &appender_file_id_len,
&tracker_obj, &storage_obj);
if (result == FAILURE)
{
logError("file: "__FILE__", line: %d, " \
"zend_parse_parameters fail!", __LINE__);
pContext->err_no = EINVAL;
RETURN_BOOL(false);
}
snprintf(new_file_id, sizeof(new_file_id), "%s", appender_file_id);
pSeperator = strchr(new_file_id, FDFS_FILE_ID_SEPERATOR);
if (pSeperator == NULL)
{
logError("file: "__FILE__", line: %d, "
"appender_file_id is invalid, "
"appender_file_id=%s",
__LINE__, appender_file_id);
pContext->err_no = EINVAL;
RETURN_BOOL(false);
}
*pSeperator = '\0';
group_name = new_file_id;
appender_filename = pSeperator + 1;
}
else
{
result = zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"ss|aa", &group_name, &group_name_len,
&appender_filename, &appender_filename_len,
&tracker_obj, &storage_obj);
if (result == FAILURE)
{
logError("file: "__FILE__", line: %d, "
"zend_parse_parameters fail!", __LINE__);
pContext->err_no = EINVAL;
RETURN_BOOL(false);
}
}
if (tracker_obj == NULL)
{
pTrackerServer = tracker_get_connection_no_pool( \
pContext->pTrackerGroup);
if (pTrackerServer == NULL)
{
pContext->err_no = ENOENT;
RETURN_BOOL(false);
}
saved_tracker_sock = -1;
tracker_hash = NULL;
}
else
{
pTrackerServer = &tracker_server;
tracker_hash = Z_ARRVAL_P(tracker_obj);
if ((result=php_fdfs_get_server_from_hash(tracker_hash, \
pTrackerServer)) != 0)
{
pContext->err_no = result;
RETURN_BOOL(false);
}
saved_tracker_sock = pTrackerServer->sock;
}
if (storage_obj == NULL)
{
pStorageServer = NULL;
storage_hash = NULL;
saved_storage_sock = -1;
}
else
{
pStorageServer = &storage_server;
storage_hash = Z_ARRVAL_P(storage_obj);
if ((result=php_fdfs_get_server_from_hash(storage_hash, \
pStorageServer)) != 0)
{
pContext->err_no = result;
RETURN_BOOL(false);
}
saved_storage_sock = pStorageServer->sock;
}
pContext->err_no = storage_regenerate_appender_filename(pTrackerServer,
pStorageServer, group_name, appender_filename,
new_group_name, new_remote_filename);
if (tracker_hash != NULL && pTrackerServer->sock != \
saved_tracker_sock)
{
CLEAR_HASH_SOCK_FIELD(tracker_hash)
}
if (pStorageServer != NULL && pStorageServer->sock != \
saved_storage_sock)
{
CLEAR_HASH_SOCK_FIELD(storage_hash)
}
if (pContext->err_no != 0)
{
RETURN_BOOL(false);
}
if (bFileId)
{
char file_id[FDFS_GROUP_NAME_MAX_LEN + 128];
int file_id_len;
file_id_len = sprintf(file_id, "%s%c%s", new_group_name,
FDFS_FILE_ID_SEPERATOR, new_remote_filename);
ZEND_RETURN_STRINGL(file_id, file_id_len, 1);
}
else
{
array_init(return_value);
zend_add_assoc_stringl_ex(return_value, "group_name",
sizeof("group_name"), new_group_name,
strlen(new_group_name), 1);
zend_add_assoc_stringl_ex(return_value, "filename",
sizeof("filename"), new_remote_filename,
strlen(new_remote_filename), 1);
}
}
static void php_fdfs_storage_set_metadata_impl(INTERNAL_FUNCTION_PARAMETERS, \
FDFSPhpContext *pContext, const bool bFileId)
{
@ -4271,7 +4462,7 @@ static void php_fdfs_http_gen_token_impl(INTERNAL_FUNCTION_PARAMETERS, \
if (argc != 2)
{
logError("file: "__FILE__", line: %d, " \
"storage_upload_file parameters " \
"fdfs_http_gen_token parameters " \
"count: %d != 2", __LINE__, argc);
pContext->err_no = EINVAL;
RETURN_BOOL(false);
@ -4359,7 +4550,7 @@ static void php_fdfs_get_file_info_impl(INTERNAL_FUNCTION_PARAMETERS, \
if (argc != param_count)
{
logError("file: "__FILE__", line: %d, " \
"storage_upload_file parameters " \
"fdfs_get_file_info parameters " \
"count: %d != %d", __LINE__, argc, param_count);
pContext->err_no = EINVAL;
RETURN_BOOL(false);
@ -4416,16 +4607,20 @@ static void php_fdfs_get_file_info_impl(INTERNAL_FUNCTION_PARAMETERS, \
}
array_init(return_value);
zend_add_assoc_long_ex(return_value, "source_id", \
zend_add_assoc_bool_ex(return_value, "get_from_server",
sizeof("get_from_server"), file_info.get_from_server);
zend_add_assoc_long_ex(return_value, "file_type",
sizeof("file_type"), file_info.file_type);
zend_add_assoc_long_ex(return_value, "source_id",
sizeof("source_id"), file_info.source_id);
zend_add_assoc_long_ex(return_value, "create_timestamp", \
zend_add_assoc_long_ex(return_value, "create_timestamp",
sizeof("create_timestamp"), file_info.create_timestamp);
zend_add_assoc_long_ex(return_value, "file_size", \
zend_add_assoc_long_ex(return_value, "file_size",
sizeof("file_size"), (long)file_info.file_size);
zend_add_assoc_stringl_ex(return_value, "source_ip_addr", \
sizeof("source_ip_addr"), file_info.source_ip_addr, \
zend_add_assoc_stringl_ex(return_value, "source_ip_addr",
sizeof("source_ip_addr"), file_info.source_ip_addr,
strlen(file_info.source_ip_addr), 1);
zend_add_assoc_long_ex(return_value, "crc32", \
zend_add_assoc_long_ex(return_value, "crc32",
sizeof("crc32"), file_info.crc32);
}
@ -4453,7 +4648,7 @@ static void php_fdfs_gen_slave_filename_impl(INTERNAL_FUNCTION_PARAMETERS, \
if (argc != 2 && argc != 3)
{
logError("file: "__FILE__", line: %d, " \
"storage_upload_file parameters " \
"fdfs_gen_slave_filename parameters " \
"count: %d != 2 or 3", __LINE__, argc);
pContext->err_no = EINVAL;
RETURN_BOOL(false);
@ -4940,7 +5135,7 @@ ZEND_FUNCTION(fastdfs_storage_modify_by_callback)
/*
boolean fastdfs_storage_modify_by_callback1(array callback_array,
long file_offset, string group_name, string appender_filename
long file_offset, string appender_file_id,
[, array tracker_server, array storage_server])
return true for success, false for error
*/
@ -4950,6 +5145,28 @@ ZEND_FUNCTION(fastdfs_storage_modify_by_callback1)
&php_context, FDFS_UPLOAD_BY_CALLBACK, true);
}
/*
boolean fastdfs_storage_regenerate_appender_filename(string group_name,
string appender_filename, [array tracker_server, array storage_server])
return assoc array for success, false for error
*/
ZEND_FUNCTION(fastdfs_storage_regenerate_appender_filename)
{
php_fdfs_storage_regenerate_appender_filename_impl(
INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, false);
}
/*
boolean fastdfs_storage_regenerate_appender_filename1(
string appender_file_id, [array tracker_server, array storage_server])
return regenerated file id for success, false for error
*/
ZEND_FUNCTION(fastdfs_storage_regenerate_appender_filename1)
{
php_fdfs_storage_regenerate_appender_filename_impl(
INTERNAL_FUNCTION_PARAM_PASSTHRU, &php_context, true);
}
/*
array fastdfs_storage_upload_appender_by_filename(string local_filename,
[string file_ext_name, string meta_list, string group_name,
@ -5962,6 +6179,36 @@ PHP_METHOD(FastDFS, storage_modify_by_callback1)
&(i_obj->context), FDFS_UPLOAD_BY_CALLBACK, true);
}
/*
boolean fastdfs_storage_regenerate_appender_filename(string group_name,
string appender_filename, [array tracker_server, array storage_server])
return assoc array for success, false for error
*/
PHP_METHOD(FastDFS, storage_regenerate_appender_filename)
{
zval *object = getThis();
php_fdfs_t *i_obj;
i_obj = (php_fdfs_t *) fdfs_get_object(object);
php_fdfs_storage_regenerate_appender_filename_impl(
INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), false);
}
/*
boolean fastdfs_storage_regenerate_appender_filename1(
string appender_file_id, [array tracker_server, array storage_server])
return regenerated file id for success, false for error
*/
PHP_METHOD(FastDFS, storage_regenerate_appender_filename1)
{
zval *object = getThis();
php_fdfs_t *i_obj;
i_obj = (php_fdfs_t *) fdfs_get_object(object);
php_fdfs_storage_regenerate_appender_filename_impl(
INTERNAL_FUNCTION_PARAM_PASSTHRU, &(i_obj->context), true);
}
/*
array FastDFS::storage_upload_appender_by_filename(string local_filename,
[string file_ext_name, string meta_list, string group_name,
@ -6758,6 +7005,19 @@ ZEND_ARG_INFO(0, tracker_server)
ZEND_ARG_INFO(0, storage_server)
ZEND_END_ARG_INFO()
ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_regenerate_appender_filename, 0, 0, 2)
ZEND_ARG_INFO(0, group_name)
ZEND_ARG_INFO(0, appender_filename)
ZEND_ARG_INFO(0, tracker_server)
ZEND_ARG_INFO(0, storage_server)
ZEND_END_ARG_INFO()
ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_regenerate_appender_filename1, 0, 0, 1)
ZEND_ARG_INFO(0, appender_file_id)
ZEND_ARG_INFO(0, tracker_server)
ZEND_ARG_INFO(0, storage_server)
ZEND_END_ARG_INFO()
ZEND_BEGIN_ARG_INFO_EX(arginfo_storage_upload_appender_by_filename, 0, 0, 1)
ZEND_ARG_INFO(0, local_filename)
ZEND_ARG_INFO(0, file_ext_name)
@ -7073,6 +7333,8 @@ static zend_function_entry fdfs_class_methods[] = {
FDFS_ME(storage_modify_by_filebuff1, arginfo_storage_modify_by_filebuff1)
FDFS_ME(storage_modify_by_callback, arginfo_storage_modify_by_callback)
FDFS_ME(storage_modify_by_callback1, arginfo_storage_modify_by_callback1)
FDFS_ME(storage_regenerate_appender_filename, arginfo_storage_regenerate_appender_filename)
FDFS_ME(storage_regenerate_appender_filename1, arginfo_storage_regenerate_appender_filename1)
FDFS_ME(storage_upload_appender_by_filename, arginfo_storage_upload_appender_by_filename)
FDFS_ME(storage_upload_appender_by_filename1, arginfo_storage_upload_appender_by_filename1)
FDFS_ME(storage_upload_appender_by_filebuff, arginfo_storage_upload_appender_by_filebuff)
@ -7539,6 +7801,13 @@ PHP_MINIT_FUNCTION(fastdfs_client)
REGISTER_LONG_CONSTANT("FDFS_STORAGE_STATUS_NONE", \
FDFS_STORAGE_STATUS_NONE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("FDFS_FILE_TYPE_NORMAL",
FDFS_FILE_TYPE_NORMAL, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("FDFS_FILE_TYPE_SLAVE",
FDFS_FILE_TYPE_SLAVE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("FDFS_FILE_TYPE_APPENDER",
FDFS_FILE_TYPE_APPENDER, CONST_CS | CONST_PERSISTENT);
return SUCCESS;
}

2
php_client/fastdfs_client.h
View File

@ -88,6 +88,8 @@ ZEND_FUNCTION(fastdfs_storage_modify_by_callback);
ZEND_FUNCTION(fastdfs_storage_modify_by_callback1);
ZEND_FUNCTION(fastdfs_storage_truncate_file);
ZEND_FUNCTION(fastdfs_storage_truncate_file1);
ZEND_FUNCTION(fastdfs_storage_regenerate_appender_filename);
ZEND_FUNCTION(fastdfs_storage_regenerate_appender_filename1);
PHP_FASTDFS_API zend_class_entry *php_fdfs_get_ce(void);
PHP_FASTDFS_API zend_class_entry *php_fdfs_get_exception(void);

32
setup.sh Executable file
View File

@ -0,0 +1,32 @@
if [ -n "$1" ]; then
TARGET_CONF_PATH=$1
else
TARGET_CONF_PATH=/etc/fdfs
fi
mkdir -p $TARGET_CONF_PATH
if [ ! -f $TARGET_CONF_PATH/tracker.conf ]; then
cp -f conf/tracker.conf $TARGET_CONF_PATH/tracker.conf
fi
if [ ! -f $TARGET_CONF_PATH/storage.conf ]; then
cp -f conf/storage.conf $TARGET_CONF_PATH/storage.conf
fi
if [ ! -f $TARGET_CONF_PATH/client.conf ]; then
cp -f conf/client.conf $TARGET_CONF_PATH/client.conf
fi
if [ ! -f $TARGET_CONF_PATH/storage_ids.conf ]; then
cp -f conf/storage_ids.conf $TARGET_CONF_PATH/storage_ids.conf
fi
if [ ! -f $TARGET_CONF_PATH/http.conf ]; then
cp -f conf/http.conf $TARGET_CONF_PATH/http.conf
fi
if [ ! -f $TARGET_CONF_PATH/mime.types ]; then
cp -f conf/mime.types $TARGET_CONF_PATH/mime.types
fi

390
storage/fdfs_storaged.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -49,14 +49,21 @@
#include "storage_dump.h"
#endif
#define ACCEPT_STAGE_NONE 0
#define ACCEPT_STAGE_DOING 1
#define ACCEPT_STAGE_DONE 2
static bool bTerminateFlag = false;
static bool bAcceptEndFlag = false;
static char accept_stage = ACCEPT_STAGE_NONE;
static void sigQuitHandler(int sig);
static void sigHupHandler(int sig);
static void sigUsrHandler(int sig);
static void sigAlarmHandler(int sig);
static int setupSchedules(pthread_t *schedule_tid);
static int setupSignalHandlers();
#if defined(DEBUG_FLAG)
/*
@ -68,24 +75,22 @@ static void sigSegvHandler(int signum, siginfo_t *info, void *ptr);
static void sigDumpHandler(int sig);
#endif
#define SCHEDULE_ENTRIES_MAX_COUNT 9
static void usage(const char *program)
{
fprintf(stderr, "Usage: %s <config_file> [start | stop | restart]\n",
fprintf(stderr, "FastDFS server v%d.%02d\n"
"Usage: %s <config_file> [start | stop | restart]\n",
g_fdfs_version.major, g_fdfs_version.minor,
program);
}
int main(int argc, char *argv[])
{
char *conf_filename;
char *action;
int result;
int sock;
int wait_count;
pthread_t schedule_tid;
struct sigaction act;
ScheduleEntry scheduleEntries[SCHEDULE_ENTRIES_MAX_COUNT];
ScheduleArray scheduleArray;
char pidFilename[MAX_PATH_SIZE];
bool stop;
@ -99,9 +104,21 @@ int main(int argc, char *argv[])
g_up_time = g_current_time;
log_init2();
trunk_shared_init();
if ((result=trunk_shared_init()) != 0)
{
log_destroy();
return result;
}
conf_filename = argv[1];
if (!fileExists(conf_filename))
{
if (starts_with(conf_filename, "-"))
{
usage(argv[0]);
return 0;
}
}
if ((result=get_base_path_from_conf_file(conf_filename,
g_fdfs_base_path, sizeof(g_fdfs_base_path))) != 0)
{
@ -109,9 +126,15 @@ int main(int argc, char *argv[])
return result;
}
if ((result=storage_check_and_make_global_data_path()) != 0)
{
log_destroy();
return result;
}
snprintf(pidFilename, sizeof(pidFilename),
"%s/data/fdfs_storaged.pid", g_fdfs_base_path);
if ((result=process_action(pidFilename, argv[2], &stop)) != 0)
action = argc >= 3 ? argv[2] : "start";
if ((result=process_action(pidFilename, action, &stop)) != 0)
{
if (result == EINVAL)
{
@ -145,6 +168,13 @@ int main(int argc, char *argv[])
return result;
}
if ((result=setupSignalHandlers()) != 0)
{
logCrit("exit abnormally!\n");
log_destroy();
return result;
}
memset(g_bind_addr, 0, sizeof(g_bind_addr));
if ((result=storage_func_init(conf_filename, \
g_bind_addr, sizeof(g_bind_addr))) != 0)
@ -204,84 +234,6 @@ int main(int argc, char *argv[])
return result;
}
memset(&act, 0, sizeof(act));
sigemptyset(&act.sa_mask);
act.sa_handler = sigUsrHandler;
if(sigaction(SIGUSR1, &act, NULL) < 0 || \
sigaction(SIGUSR2, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
logCrit("exit abnormally!\n");
return errno;
}
act.sa_handler = sigHupHandler;
if(sigaction(SIGHUP, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
logCrit("exit abnormally!\n");
return errno;
}
act.sa_handler = SIG_IGN;
if(sigaction(SIGPIPE, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
logCrit("exit abnormally!\n");
return errno;
}
act.sa_handler = sigQuitHandler;
if(sigaction(SIGINT, &act, NULL) < 0 || \
sigaction(SIGTERM, &act, NULL) < 0 || \
sigaction(SIGQUIT, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
logCrit("exit abnormally!\n");
return errno;
}
#if defined(DEBUG_FLAG)
/*
#if defined(OS_LINUX)
memset(&act, 0, sizeof(act));
act.sa_sigaction = sigSegvHandler;
act.sa_flags = SA_SIGINFO;
if (sigaction(SIGSEGV, &act, NULL) < 0 || \
sigaction(SIGABRT, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
logCrit("exit abnormally!\n");
return errno;
}
#endif
*/
memset(&act, 0, sizeof(act));
sigemptyset(&act.sa_mask);
act.sa_handler = sigDumpHandler;
if(sigaction(SIGUSR1, &act, NULL) < 0 || \
sigaction(SIGUSR2, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
logCrit("exit abnormally!\n");
return errno;
}
#endif
#ifdef WITH_HTTPD
if (!g_http_params.disabled)
@ -307,80 +259,7 @@ int main(int argc, char *argv[])
return result;
}
scheduleArray.entries = scheduleEntries;
scheduleArray.count = 0;
memset(scheduleEntries, 0, sizeof(scheduleEntries));
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
g_sync_log_buff_interval, log_sync_func, &g_log_context);
scheduleArray.count++;
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
g_sync_binlog_buff_interval, fdfs_binlog_sync_func, NULL);
scheduleArray.count++;
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
g_sync_stat_file_interval, fdfs_stat_file_sync_func, NULL);
scheduleArray.count++;
if (g_if_use_trunk_file)
{
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
1, trunk_binlog_sync_func, NULL);
scheduleArray.count++;
}
if (g_use_access_log)
{
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
g_sync_log_buff_interval, log_sync_func, &g_access_log_context);
scheduleArray.count++;
if (g_rotate_access_log)
{
INIT_SCHEDULE_ENTRY_EX(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, g_access_log_rotate_time,
24 * 3600, log_notify_rotate, &g_access_log_context);
scheduleArray.count++;
if (g_log_file_keep_days > 0)
{
log_set_keep_days(&g_access_log_context,
g_log_file_keep_days);
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, 1, 0, 0, 24 * 3600,
log_delete_old_files, &g_access_log_context);
scheduleArray.count++;
}
}
}
if (g_rotate_error_log)
{
INIT_SCHEDULE_ENTRY_EX(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, g_error_log_rotate_time,
24 * 3600, log_notify_rotate, &g_log_context);
scheduleArray.count++;
if (g_log_file_keep_days > 0)
{
log_set_keep_days(&g_log_context, g_log_file_keep_days);
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, 1, 0, 0, 24 * 3600,
log_delete_old_files, &g_log_context);
scheduleArray.count++;
}
}
if ((result=sched_start(&scheduleArray, &schedule_tid, \
g_thread_stack_size, (bool * volatile)&g_continue_flag)) != 0)
if ((result=setupSchedules(&schedule_tid)) != 0)
{
logCrit("exit abnormally!\n");
log_destroy();
@ -403,10 +282,10 @@ int main(int argc, char *argv[])
log_set_cache(true);
bTerminateFlag = false;
bAcceptEndFlag = false;
accept_stage = ACCEPT_STAGE_DOING;
storage_accept_loop(sock);
bAcceptEndFlag = true;
accept_stage = ACCEPT_STAGE_DONE;
fdfs_binlog_sync_func(NULL); //binlog fsync
@ -438,7 +317,7 @@ int main(int argc, char *argv[])
*/
usleep(10000);
if (++wait_count > 6000)
if (++wait_count > 9000)
{
logWarning("waiting timeout, exit!");
break;
@ -448,7 +327,6 @@ int main(int argc, char *argv[])
tracker_report_destroy();
storage_service_destroy();
storage_sync_destroy();
storage_func_destroy();
if (g_if_use_trunk_file)
{
@ -456,6 +334,7 @@ int main(int argc, char *argv[])
storage_trunk_destroy();
}
storage_func_destroy();
delete_pid_file(pidFilename);
logInfo("exit normally.\n");
log_destroy();
@ -467,6 +346,7 @@ static void sigQuitHandler(int sig)
{
if (!bTerminateFlag)
{
tcp_set_try_again_when_interrupt(false);
set_timer(1, 1, sigAlarmHandler);
bTerminateFlag = true;
@ -482,7 +362,7 @@ static void sigAlarmHandler(int sig)
{
ConnectionInfo server;
if (bAcceptEndFlag)
if (accept_stage != ACCEPT_STAGE_DOING)
{
return;
}
@ -556,3 +436,179 @@ static void sigDumpHandler(int sig)
}
#endif
static int setupSchedules(pthread_t *schedule_tid)
{
#define SCHEDULE_ENTRIES_MAX_COUNT 10
ScheduleEntry scheduleEntries[SCHEDULE_ENTRIES_MAX_COUNT];
ScheduleArray scheduleArray;
int result;
scheduleArray.entries = scheduleEntries;
scheduleArray.count = 0;
memset(scheduleEntries, 0, sizeof(scheduleEntries));
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
g_sync_log_buff_interval, log_sync_func, &g_log_context);
scheduleArray.count++;
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
g_sync_binlog_buff_interval, fdfs_binlog_sync_func, NULL);
scheduleArray.count++;
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
g_sync_stat_file_interval, fdfs_stat_file_sync_func, NULL);
scheduleArray.count++;
if (g_if_use_trunk_file)
{
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
1, trunk_binlog_sync_func, NULL);
scheduleArray.count++;
}
if (g_use_access_log)
{
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, TIME_NONE, TIME_NONE, TIME_NONE,
g_sync_log_buff_interval, log_sync_func, &g_access_log_context);
scheduleArray.count++;
if (g_rotate_access_log)
{
INIT_SCHEDULE_ENTRY_EX(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, g_access_log_rotate_time,
24 * 3600, log_notify_rotate, &g_access_log_context);
scheduleArray.count++;
if (g_log_file_keep_days > 0)
{
log_set_keep_days(&g_access_log_context,
g_log_file_keep_days);
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, 1, 0, 0, 24 * 3600,
log_delete_old_files, &g_access_log_context);
scheduleArray.count++;
}
}
}
if (g_rotate_error_log)
{
INIT_SCHEDULE_ENTRY_EX(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, g_error_log_rotate_time,
24 * 3600, log_notify_rotate, &g_log_context);
scheduleArray.count++;
if (g_log_file_keep_days > 0)
{
log_set_keep_days(&g_log_context, g_log_file_keep_days);
INIT_SCHEDULE_ENTRY(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, 1, 0, 0, 24 * 3600,
log_delete_old_files, &g_log_context);
scheduleArray.count++;
}
}
if (g_compress_binlog)
{
INIT_SCHEDULE_ENTRY_EX(scheduleEntries[scheduleArray.count],
scheduleArray.count + 1, g_compress_binlog_time,
24 * 3600, fdfs_binlog_compress_func, NULL);
scheduleArray.count++;
}
if ((result=sched_start(&scheduleArray, schedule_tid,
g_thread_stack_size, (bool * volatile)&g_continue_flag)) != 0)
{
return result;
}
return 0;
}
static int setupSignalHandlers()
{
struct sigaction act;
memset(&act, 0, sizeof(act));
sigemptyset(&act.sa_mask);
act.sa_handler = sigUsrHandler;
if(sigaction(SIGUSR1, &act, NULL) < 0 || \
sigaction(SIGUSR2, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
return errno != 0 ? errno : EFAULT;
}
act.sa_handler = sigHupHandler;
if(sigaction(SIGHUP, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
return errno != 0 ? errno : EFAULT;
}
act.sa_handler = SIG_IGN;
if(sigaction(SIGPIPE, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
return errno != 0 ? errno : EFAULT;
}
act.sa_handler = sigQuitHandler;
if(sigaction(SIGINT, &act, NULL) < 0 || \
sigaction(SIGTERM, &act, NULL) < 0 || \
sigaction(SIGQUIT, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
return errno != 0 ? errno : EFAULT;
}
#if defined(DEBUG_FLAG)
/*
#if defined(OS_LINUX)
memset(&act, 0, sizeof(act));
act.sa_sigaction = sigSegvHandler;
act.sa_flags = SA_SIGINFO;
if (sigaction(SIGSEGV, &act, NULL) < 0 || \
sigaction(SIGABRT, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
return errno != 0 ? errno : EFAULT;
}
#endif
*/
memset(&act, 0, sizeof(act));
sigemptyset(&act.sa_mask);
act.sa_handler = sigDumpHandler;
if(sigaction(SIGUSR1, &act, NULL) < 0 || \
sigaction(SIGUSR2, &act, NULL) < 0)
{
logCrit("file: "__FILE__", line: %d, " \
"call sigaction fail, errno: %d, error info: %s", \
__LINE__, errno, STRERROR(errno));
return errno != 0 ? errno : EFAULT;
}
#endif
return 0;
}

2
storage/fdht_client/fdht_client.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>

2
storage/fdht_client/fdht_client.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdht_client.h

2
storage/fdht_client/fdht_define.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdht_define.h

2
storage/fdht_client/fdht_func.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdht_func.c

2
storage/fdht_client/fdht_func.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdht_func.h

2
storage/fdht_client/fdht_global.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <netdb.h>

2
storage/fdht_client/fdht_global.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdht_global.h

2
storage/fdht_client/fdht_proto.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <sys/types.h>

2
storage/fdht_client/fdht_proto.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdht_proto.h

2
storage/fdht_client/fdht_proto_types.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdht_proto_types.h

2
storage/fdht_client/fdht_types.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//fdht_types.h

41
storage/storage_dio.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -24,6 +24,7 @@
#include "fastcommon/pthread_func.h"
#include "fastcommon/logger.h"
#include "fastcommon/sockopt.h"
#include "fastcommon/ioevent_loop.h"
#include "storage_dio.h"
#include "storage_nio.h"
#include "storage_service.h"
@ -156,7 +157,7 @@ int storage_dio_queue_push(struct fast_task_info *pTask)
pClientInfo->stage |= FDFS_STORAGE_STAGE_DIO_THREAD;
if ((result=blocked_queue_push(&(pContext->queue), pTask)) != 0)
{
add_to_deleted_list(pTask);
ioevent_add_to_deleted_list(pTask);
return result;
}
@ -222,7 +223,7 @@ int dio_delete_trunk_file(struct fast_task_info *pTask)
pFileContext = &(((StorageClientInfo *)pTask->arg)->file_context);
if ((result=trunk_file_delete(pFileContext->filename, \
if ((result=trunk_file_delete(pFileContext->filename,
&(pFileContext->extra_info.upload.trunk_info))) != 0)
{
pFileContext->log_callback(pTask, result);
@ -245,7 +246,7 @@ int dio_discard_file(struct fast_task_info *pTask)
else
{
pFileContext->buff_offset = 0;
storage_nio_notify(pTask); //notify nio to deal
pFileContext->continue_callback(pTask);
}
return 0;
@ -353,6 +354,12 @@ int dio_read_file(struct fast_task_info *pTask)
break;
}
if (pFileContext->calc_crc32)
{
pFileContext->crc32 = CRC32_ex(pTask->data + pTask->length,
read_bytes, pFileContext->crc32);
}
pTask->length += read_bytes;
pFileContext->offset += read_bytes;
@ -363,7 +370,7 @@ int dio_read_file(struct fast_task_info *pTask)
if (pFileContext->offset < pFileContext->end)
{
storage_nio_notify(pTask); //notify nio to deal
pFileContext->continue_callback(pTask);
}
else
{
@ -371,6 +378,11 @@ int dio_read_file(struct fast_task_info *pTask)
close(pFileContext->fd);
pFileContext->fd = -1;
if (pFileContext->calc_crc32)
{
pFileContext->crc32 = CRC32_FINAL( \
pFileContext->crc32);
}
pFileContext->done_callback(pTask, result);
}
@ -475,7 +487,7 @@ int dio_write_file(struct fast_task_info *pTask)
if (pFileContext->offset < pFileContext->end)
{
pFileContext->buff_offset = 0;
storage_nio_notify(pTask); //notify nio to deal
pFileContext->continue_callback(pTask);
}
else
{
@ -803,35 +815,34 @@ int dio_check_trunk_file_ex(int fd, const char *filename, const int64_t offset)
{
int result;
char old_header[FDFS_TRUNK_FILE_HEADER_SIZE];
char expect_header[FDFS_TRUNK_FILE_HEADER_SIZE];
static char expect_header[FDFS_TRUNK_FILE_HEADER_SIZE] = {'\0'};
if (fc_safe_read(fd, old_header, FDFS_TRUNK_FILE_HEADER_SIZE) !=
FDFS_TRUNK_FILE_HEADER_SIZE)
{
result = errno != 0 ? errno : EIO;
logError("file: "__FILE__", line: %d, " \
"read trunk header of file: %s fail, " \
"errno: %d, error info: %s", \
__LINE__, filename, \
logError("file: "__FILE__", line: %d, "
"read trunk header of file: %s fail, "
"errno: %d, error info: %s",
__LINE__, filename,
result, STRERROR(result));
return result;
}
memset(expect_header, 0, sizeof(expect_header));
if (memcmp(old_header, expect_header, \
if (memcmp(old_header, expect_header,
FDFS_TRUNK_FILE_HEADER_SIZE) != 0)
{
FDFSTrunkHeader srcOldTrunkHeader;
FDFSTrunkHeader newOldTrunkHeader;
trunk_unpack_header(old_header, &srcOldTrunkHeader);
memcpy(&newOldTrunkHeader, &srcOldTrunkHeader, \
memcpy(&newOldTrunkHeader, &srcOldTrunkHeader,
sizeof(FDFSTrunkHeader));
newOldTrunkHeader.alloc_size = 0;
newOldTrunkHeader.file_size = 0;
newOldTrunkHeader.file_type = 0;
trunk_pack_header(&newOldTrunkHeader, old_header);
if (memcmp(old_header, expect_header, \
if (memcmp(old_header, expect_header,
FDFS_TRUNK_FILE_HEADER_SIZE) != 0)
{
char buff[256];

2
storage/storage_dio.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_dio.h

1298
storage/storage_disk_recovery.c
View File

File diff suppressed because it is too large Load Diff

6
storage/storage_disk_recovery.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_disk_recovery.h
@ -18,8 +18,8 @@
extern "C" {
#endif
int storage_disk_recovery_start(const int store_path_index);
int storage_disk_recovery_restore(const char *pBasePath);
int storage_disk_recovery_prepare(const int store_path_index);
int storage_disk_recovery_check_restore(const char *pBasePath);
#ifdef __cplusplus
}

8
storage/storage_dump.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -246,9 +246,9 @@ static int fdfs_dump_global_vars(char *buff, const int buffSize)
total_len += snprintf(buff + total_len, buffSize - total_len,
"\tg_fdfs_store_paths.paths[%d]=%s, " \
"total=%d MB, free=%d MB\n", i, \
g_fdfs_store_paths.paths[i], \
g_path_space_list[i].total_mb, \
g_path_space_list[i].free_mb);
g_fdfs_store_paths.paths[i].path, \
g_fdfs_store_paths.paths[i].total_mb, \
g_fdfs_store_paths.paths[i].free_mb);
}
if (total_len < buffSize - 1)

2
storage/storage_dump.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_dump.h

576
storage/storage_func.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_func.c
@ -51,8 +51,18 @@
#include "fdfs_http_shared.h"
#endif
typedef struct
{
char ip_addr[IP_ADDRESS_SIZE];
short port;
unsigned char store_path_index;
char padding;
int create_time;
} FDFSStorePathMarkInfo;
#define DATA_DIR_INITED_FILENAME ".data_init_flag"
#define STORAGE_STAT_FILENAME "storage_stat.dat"
#define STORE_PATH_MARK_FILENAME ".fastdfs_vars"
#define INIT_ITEM_STORAGE_JOIN_TIME "storage_join_time"
#define INIT_ITEM_SYNC_OLD_DONE "sync_old_done"
@ -63,6 +73,9 @@
#define INIT_ITEM_LAST_HTTP_PORT "last_http_port"
#define INIT_ITEM_CURRENT_TRUNK_FILE_ID "current_trunk_file_id"
#define INIT_ITEM_TRUNK_LAST_COMPRESS_TIME "trunk_last_compress_time"
#define INIT_ITEM_TRUNK_BINLOG_COMPRESS_STAGE \
"trunk_binlog_compress_stage"
#define INIT_ITEM_STORE_PATH_MARK_PREFIX "store_path_mark"
#define STAT_ITEM_TOTAL_UPLOAD "total_upload_count"
#define STAT_ITEM_SUCCESS_UPLOAD "success_upload_count"
@ -210,7 +223,7 @@ static int tracker_get_my_server_id()
struct in_addr ip_addr;
char ip_str[256];
if (inet_pton(AF_INET, g_tracker_client_ip.ips[0], &ip_addr) == 1)
if (inet_pton(AF_INET, g_tracker_client_ip.ips[0].address, &ip_addr) == 1)
{
g_server_id_in_filename = ip_addr.s_addr;
}
@ -218,7 +231,7 @@ static int tracker_get_my_server_id()
{
logError("file: "__FILE__", line: %d, " \
"call inet_pton for ip: %s fail", \
__LINE__, g_tracker_client_ip.ips[0]);
__LINE__, g_tracker_client_ip.ips[0].address);
g_server_id_in_filename = INADDR_NONE;
}
@ -234,7 +247,7 @@ static int tracker_get_my_server_id()
}
result = tracker_get_storage_id(pTrackerServer,
g_group_name, g_tracker_client_ip.ips[0],
g_group_name, g_tracker_client_ip.ips[0].address,
g_my_server_id_str);
tracker_close_connection_ex(pTrackerServer, result != 0);
if (result != 0)
@ -250,7 +263,7 @@ static int tracker_get_my_server_id()
else
{
snprintf(g_my_server_id_str, sizeof(g_my_server_id_str), "%s",
g_tracker_client_ip.ips[0]);
g_tracker_client_ip.ips[0].address);
}
fdfs_multi_ips_to_string(&g_tracker_client_ip,
@ -628,62 +641,337 @@ int storage_write_to_stat_file()
int storage_write_to_sync_ini_file()
{
char full_filename[MAX_PATH_SIZE];
char buff[512];
char buff[4 * 1024];
char ip_str[256];
int fd;
int len;
int result;
int i;
snprintf(full_filename, sizeof(full_filename), \
snprintf(full_filename, sizeof(full_filename),
"%s/data/%s", g_fdfs_base_path, DATA_DIR_INITED_FILENAME);
if ((fd=open(full_filename, O_WRONLY | O_CREAT | O_TRUNC, 0644)) < 0)
{
logError("file: "__FILE__", line: %d, " \
"open file \"%s\" fail, " \
"errno: %d, error info: %s", \
__LINE__, full_filename, \
errno, STRERROR(errno));
return errno != 0 ? errno : ENOENT;
}
fdfs_multi_ips_to_string(&g_tracker_client_ip,
ip_str, sizeof(ip_str));
len = sprintf(buff, "%s=%d\n" \
"%s=%d\n" \
"%s=%s\n" \
"%s=%d\n" \
"%s=%s\n" \
"%s=%d\n" \
"%s=%d\n" \
"%s=%d\n" \
"%s=%d\n", \
INIT_ITEM_STORAGE_JOIN_TIME, g_storage_join_time, \
INIT_ITEM_SYNC_OLD_DONE, g_sync_old_done, \
INIT_ITEM_SYNC_SRC_SERVER, g_sync_src_id, \
INIT_ITEM_SYNC_UNTIL_TIMESTAMP, g_sync_until_timestamp, \
INIT_ITEM_LAST_IP_ADDRESS, ip_str, \
INIT_ITEM_LAST_SERVER_PORT, g_last_server_port, \
len = sprintf(buff, "%s=%d\n"
"%s=%d\n"
"%s=%s\n"
"%s=%d\n"
"%s=%s\n"
"%s=%d\n"
"%s=%d\n"
"%s=%u\n"
"%s=%d\n"
"%s=%d\n",
INIT_ITEM_STORAGE_JOIN_TIME, g_storage_join_time,
INIT_ITEM_SYNC_OLD_DONE, g_sync_old_done,
INIT_ITEM_SYNC_SRC_SERVER, g_sync_src_id,
INIT_ITEM_SYNC_UNTIL_TIMESTAMP, g_sync_until_timestamp,
INIT_ITEM_LAST_IP_ADDRESS, ip_str,
INIT_ITEM_LAST_SERVER_PORT, g_last_server_port,
INIT_ITEM_LAST_HTTP_PORT, g_last_http_port,
INIT_ITEM_CURRENT_TRUNK_FILE_ID, g_current_trunk_file_id, \
INIT_ITEM_TRUNK_LAST_COMPRESS_TIME, (int)g_trunk_last_compress_time
);
if (fc_safe_write(fd, buff, len) != len)
INIT_ITEM_CURRENT_TRUNK_FILE_ID, g_current_trunk_file_id,
INIT_ITEM_TRUNK_LAST_COMPRESS_TIME,
(int)g_trunk_last_compress_time,
INIT_ITEM_TRUNK_BINLOG_COMPRESS_STAGE,
g_trunk_binlog_compress_stage);
if (g_check_store_path_mark)
{
logError("file: "__FILE__", line: %d, " \
"write to file \"%s\" fail, " \
"errno: %d, error info: %s", \
__LINE__, full_filename, \
errno, STRERROR(errno));
close(fd);
return errno != 0 ? errno : EIO;
for (i=0; i<g_fdfs_store_paths.count; i++)
{
if (g_fdfs_store_paths.paths[i].mark != NULL)
{
len += sprintf(buff + len, "%s%d=%s\n",
INIT_ITEM_STORE_PATH_MARK_PREFIX, i,
g_fdfs_store_paths.paths[i].mark);
}
}
}
close(fd);
if ((result=safeWriteToFile(full_filename, buff, len)) != 0)
{
return result;
}
STORAGE_CHOWN(full_filename, geteuid(), getegid())
return 0;
}
int storage_check_and_make_global_data_path()
{
char data_path[MAX_PATH_SIZE];
snprintf(data_path, sizeof(data_path), "%s/data",
g_fdfs_base_path);
if (!fileExists(data_path))
{
if (mkdir(data_path, 0755) != 0)
{
logError("file: "__FILE__", line: %d, "
"mkdir \"%s\" fail, "
"errno: %d, error info: %s",
__LINE__, data_path,
errno, STRERROR(errno));
return errno != 0 ? errno : EPERM;
}
STORAGE_CHOWN(data_path, geteuid(), getegid())
}
return 0;
}
static int storage_load_store_path_marks(IniContext *pIniContext)
{
char *pValue;
char name[64];
int i;
if (!g_check_store_path_mark)
{
return 0;
}
for (i=0; i<g_fdfs_store_paths.count; i++)
{
sprintf(name, "%s%d", INIT_ITEM_STORE_PATH_MARK_PREFIX, i);
pValue = iniGetStrValue(NULL, name, pIniContext);
if (pValue != NULL)
{
g_fdfs_store_paths.paths[i].mark = strdup(pValue);
if (g_fdfs_store_paths.paths[i].mark == NULL)
{
logError("file: "__FILE__", line: %d, "
"strdup %s fail", __LINE__, pValue);
return errno != 0 ? errno : ENOMEM;
}
}
}
return 0;
}
static int storage_generate_store_path_mark(const int store_path_index)
{
FDFSStorePathMarkInfo mark_info;
char full_filename[MAX_PATH_SIZE];
char buff[256];
char *mark_str;
int result;
int mark_len;
int buff_len;
int bytes;
bytes = sizeof(FDFSStorePathMarkInfo) * 4 / 3;
mark_str = (char *)malloc(bytes);
if (mark_str == NULL)
{
logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail, "
"errno: %d, error info: %s", __LINE__,
bytes, errno, STRERROR(errno));
return errno != 0 ? errno : ENOMEM;
}
memset(&mark_info, 0, sizeof(FDFSStorePathMarkInfo));
strcpy(mark_info.ip_addr, g_tracker_client_ip.ips[0].address);
mark_info.port = g_server_port;
mark_info.store_path_index = store_path_index;
mark_info.create_time = g_current_time;
base64_encode_ex(&g_fdfs_base64_context, (char *)&mark_info,
sizeof(mark_info), mark_str, &mark_len, false);
snprintf(full_filename, sizeof(full_filename), "%s/data/%s",
g_fdfs_store_paths.paths[store_path_index].path,
STORE_PATH_MARK_FILENAME);
buff_len = sprintf(buff, "%s=%s\n",
INIT_ITEM_STORE_PATH_MARK_PREFIX, mark_str);
if ((result=safeWriteToFile(full_filename, buff, buff_len)) != 0)
{
free(mark_str);
return result;
}
if (g_fdfs_store_paths.paths[store_path_index].mark != NULL)
{
free(g_fdfs_store_paths.paths[store_path_index].mark);
}
g_fdfs_store_paths.paths[store_path_index].mark = mark_str;
return 0;
}
static int storage_check_store_path_mark(const int store_path_index,
const bool bPathCreated)
{
char full_filename[MAX_PATH_SIZE];
char *mark;
int result;
if (!g_check_store_path_mark)
{
return 0;
}
snprintf(full_filename, sizeof(full_filename), "%s/data/%s",
g_fdfs_store_paths.paths[store_path_index].path,
STORE_PATH_MARK_FILENAME);
if (fileExists(full_filename))
{
IniContext iniContext;
char *pValue;
if ((result=iniLoadFromFile(full_filename, &iniContext)) != 0)
{
logError("file: "__FILE__", line: %d, "
"load from file \"%s\" fail, "
"error code: %d", __LINE__,
full_filename, result);
return result;
}
pValue = iniGetStrValue(NULL, INIT_ITEM_STORE_PATH_MARK_PREFIX,
&iniContext);
if (pValue != NULL)
{
mark = strdup(pValue);
if (mark == NULL)
{
logError("file: "__FILE__", line: %d, "
"strdup %s fail", __LINE__, pValue);
iniFreeContext(&iniContext);
return errno != 0 ? errno : ENOMEM;
}
}
else
{
mark = NULL;
}
iniFreeContext(&iniContext);
}
else
{
mark = NULL;
}
if (g_fdfs_store_paths.paths[store_path_index].mark == NULL)
{
if (mark != NULL)
{
logCrit("file: "__FILE__", line: %d, "
"the store path #%d: %s maybe used by other "
"storage server. if you confirm that it is NOT "
"used by other storage server, you can delete "
"the mark file %s then try again", __LINE__,
store_path_index, g_fdfs_store_paths.
paths[store_path_index].path, full_filename);
free(mark);
return EINVAL;
}
}
else
{
if (mark != NULL)
{
if (strcmp(g_fdfs_store_paths.paths[store_path_index].mark,
mark) == 0)
{
free(mark);
return 0;
}
else
{
FDFSStorePathMarkInfo mark_info;
int mark_len;
int dest_len;
mark_len = strlen(mark);
dest_len = sizeof(FDFSStorePathMarkInfo) * 4 / 3;
if (mark_len != dest_len)
{
logError("file: "__FILE__", line: %d, "
"the mark string is not base64 encoded, "
"store path #%d, the mark file: %s, "
"the mark string: %s", __LINE__,
store_path_index, full_filename, mark);
memset(&mark_info, 0, sizeof(FDFSStorePathMarkInfo));
}
else if (base64_decode_auto(&g_fdfs_base64_context,
mark, mark_len, (char *)&mark_info,
&dest_len) == NULL)
{
logError("file: "__FILE__", line: %d, "
"the mark string is not base64 encoded, "
"store path #%d, the mark file: %s, "
"the mark string: %s", __LINE__,
store_path_index, full_filename, mark);
memset(&mark_info, 0, sizeof(FDFSStorePathMarkInfo));
}
if (mark_info.port > 0)
{
char time_str[32];
mark_info.ip_addr[IP_ADDRESS_SIZE - 1] = '\0';
formatDatetime(mark_info.create_time,
"%Y-%m-%d %H:%M:%S",
time_str, sizeof(time_str));
logCrit("file: "__FILE__", line: %d, "
"the store path #%d: %s maybe used by other "
"storage server. fields in the mark file: "
"{ ip_addr: %s, port: %d,"
" store_path_index: %d,"
" create_time: %s }, "
"if you confirm that it is NOT "
"used by other storage server, you can delete "
"the mark file %s then try again. if you DON'T "
"really need to check store path mark to prevent "
"confusion, you can set the parameter "
"check_store_path_mark to false in storage.conf",
__LINE__, store_path_index, g_fdfs_store_paths.
paths[store_path_index].path,
mark_info.ip_addr, mark_info.port,
mark_info.store_path_index, time_str,
full_filename);
}
else
{
logCrit("file: "__FILE__", line: %d, "
"the store path #%d: %s maybe used by other "
"storage server. if you confirm that it is NOT "
"used by other storage server, you can delete "
"the mark file %s then try again", __LINE__,
store_path_index, g_fdfs_store_paths.
paths[store_path_index].path, full_filename);
}
free(mark);
return EINVAL;
}
}
else
{
if (!bPathCreated)
{
logWarning("file: "__FILE__", line: %d, "
"the mark file of store path #%d: %s is missed, "
"try to re-create the mark file: %s", __LINE__,
store_path_index, g_fdfs_store_paths.
paths[store_path_index].path, full_filename);
}
}
}
if ((result=storage_generate_store_path_mark(store_path_index)) != 0)
{
return result;
}
return storage_write_to_sync_ini_file();
}
static int storage_check_and_make_data_dirs()
{
int result;
@ -693,9 +981,9 @@ static int storage_check_and_make_data_dirs()
char error_info[256];
bool pathCreated;
snprintf(data_path, sizeof(data_path), "%s/data", \
snprintf(data_path, sizeof(data_path), "%s/data",
g_fdfs_base_path);
snprintf(full_filename, sizeof(full_filename), "%s/%s", \
snprintf(full_filename, sizeof(full_filename), "%s/%s",
data_path, DATA_DIR_INITED_FILENAME);
if (fileExists(full_filename))
{
@ -778,10 +1066,19 @@ static int storage_check_and_make_data_dirs()
g_last_http_port = atoi(pValue);
}
g_current_trunk_file_id = iniGetIntValue(NULL, \
g_current_trunk_file_id = iniGetIntValue(NULL,
INIT_ITEM_CURRENT_TRUNK_FILE_ID, &iniContext, 0);
g_trunk_last_compress_time = iniGetIntValue(NULL, \
g_trunk_last_compress_time = iniGetIntValue(NULL,
INIT_ITEM_TRUNK_LAST_COMPRESS_TIME , &iniContext, 0);
g_trunk_binlog_compress_stage = iniGetIntValue(NULL,
INIT_ITEM_TRUNK_BINLOG_COMPRESS_STAGE,
&iniContext, STORAGE_TRUNK_COMPRESS_STAGE_NONE);
if ((result=storage_load_store_path_marks(&iniContext)) != 0)
{
iniFreeContext(&iniContext);
return result;
}
iniFreeContext(&iniContext);
@ -810,7 +1107,7 @@ static int storage_check_and_make_data_dirs()
"g_last_storage_ip = %s, "
"g_last_server_port = %d, "
"g_last_http_port = %d, "
"g_current_trunk_file_id = %d, "
"g_current_trunk_file_id = %u, "
"g_trunk_last_compress_time = %d",
g_sync_old_done, g_sync_src_id, g_sync_until_timestamp,
g_last_storage_ip, g_last_server_port, g_last_http_port,
@ -820,21 +1117,10 @@ static int storage_check_and_make_data_dirs()
}
else
{
if (!fileExists(data_path))
if ((result=storage_check_and_make_global_data_path()) != 0)
{
if (mkdir(data_path, 0755) != 0)
{
logError("file: "__FILE__", line: %d, " \
"mkdir \"%s\" fail, " \
"errno: %d, error info: %s", \
__LINE__, data_path, \
errno, STRERROR(errno));
return errno != 0 ? errno : EPERM;
return result;
}
STORAGE_CHOWN(data_path, geteuid(), getegid())
}
g_last_server_port = g_server_port;
g_last_http_port = g_http_port;
g_storage_join_time = g_current_time;
@ -846,29 +1132,36 @@ static int storage_check_and_make_data_dirs()
for (i=0; i<g_fdfs_store_paths.count; i++)
{
if ((result=storage_make_data_dirs(g_fdfs_store_paths.paths[i], \
if ((result=storage_make_data_dirs(g_fdfs_store_paths.paths[i].path,
&pathCreated)) != 0)
{
return result;
}
if ((result=storage_check_store_path_mark(i, pathCreated)) != 0)
{
return result;
}
if (g_sync_old_done && pathCreated) //repair damaged disk
{
if ((result=storage_disk_recovery_start(i)) != 0)
if ((result=storage_disk_recovery_prepare(i)) != 0)
{
return result;
}
}
result = storage_disk_recovery_restore(g_fdfs_store_paths.paths[i]);
result = storage_disk_recovery_check_restore(
g_fdfs_store_paths.paths[i].path);
if (result == EAGAIN) //need to re-fetch binlog
{
if ((result=storage_disk_recovery_start(i)) != 0)
if ((result=storage_disk_recovery_prepare(i)) != 0)
{
return result;
}
result=storage_disk_recovery_restore(g_fdfs_store_paths.paths[i]);
result = storage_disk_recovery_check_restore(
g_fdfs_store_paths.paths[i].path);
}
if (result != 0)
@ -883,8 +1176,8 @@ static int storage_check_and_make_data_dirs()
static int storage_make_data_dirs(const char *pBasePath, bool *pathCreated)
{
char data_path[MAX_PATH_SIZE];
char dir_name[9];
char sub_name[9];
char dir_name[16];
char sub_name[16];
char min_sub_path[16];
char max_sub_path[16];
int i, k;
@ -919,10 +1212,11 @@ static int storage_make_data_dirs(const char *pBasePath, bool *pathCreated)
return errno != 0 ? errno : ENOENT;
}
sprintf(min_sub_path, FDFS_STORAGE_DATA_DIR_FORMAT"/"FDFS_STORAGE_DATA_DIR_FORMAT,
0, 0);
sprintf(max_sub_path, FDFS_STORAGE_DATA_DIR_FORMAT"/"FDFS_STORAGE_DATA_DIR_FORMAT,
g_subdir_count_per_path-1, g_subdir_count_per_path-1);
sprintf(min_sub_path, FDFS_STORAGE_DATA_DIR_FORMAT"/"
FDFS_STORAGE_DATA_DIR_FORMAT, 0, 0);
sprintf(max_sub_path, FDFS_STORAGE_DATA_DIR_FORMAT"/"
FDFS_STORAGE_DATA_DIR_FORMAT, g_subdir_count_per_path - 1,
g_subdir_count_per_path - 1);
if (fileExists(min_sub_path) && fileExists(max_sub_path))
{
return 0;
@ -1027,7 +1321,6 @@ static int init_fsync_pthread_cond()
static int storage_load_paths(IniContext *pItemContext)
{
int result;
int bytes;
result = storage_load_paths_from_conf_file(pItemContext);
if (result != 0)
@ -1035,16 +1328,6 @@ static int storage_load_paths(IniContext *pItemContext)
return result;
}
bytes = sizeof(FDFSStorePathInfo) * g_fdfs_store_paths.count;
g_path_space_list = (FDFSStorePathInfo *)malloc(bytes);
if (g_path_space_list == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, errno: %d, error info: %s", \
__LINE__, bytes, errno, STRERROR(errno));
return errno != 0 ? errno : ENOMEM;
}
memset(g_path_space_list, 0, bytes);
return 0;
}
@ -1084,7 +1367,7 @@ static int storage_check_tracker_ipaddr(const char *filename)
return 0;
}
static int init_my_status_per_tracker()
static int init_my_result_per_tracker()
{
int bytes;
TrackerServerInfo *pTrackerServer;
@ -1109,7 +1392,8 @@ static int init_my_status_per_tracker()
pTrackerServer++)
{
pReportStatus->my_status = -1;
pReportStatus->src_storage_status = -1;
pReportStatus->my_result = -1;
pReportStatus->src_storage_result = -1;
pReportStatus++;
}
@ -1138,13 +1422,6 @@ int storage_func_init(const char *filename, \
int64_t rotate_access_log_size;
int64_t rotate_error_log_size;
/*
while (nThreadCount > 0)
{
sleep(1);
}
*/
if ((result=iniLoadFromFile(filename, &iniContext)) != 0)
{
logError("file: "__FILE__", line: %d, " \
@ -1444,6 +1721,18 @@ int storage_func_init(const char *filename, \
break;
}
g_disk_recovery_threads = iniGetIntValue(NULL,
"disk_recovery_threads", &iniContext, 1);
if (g_disk_recovery_threads <= 0)
{
logError("file: "__FILE__", line: %d, "
"item \"disk_recovery_threads\" is invalid, "
"value: %d <= 0!", __LINE__,
g_disk_recovery_threads);
result = EINVAL;
break;
}
/*
g_disk_rw_direct = iniGetBoolValue(NULL, \
"disk_rw_direct", &iniContext, false);
@ -1726,6 +2015,30 @@ int storage_func_init(const char *filename, \
g_rotate_error_log = iniGetBoolValue(NULL, "rotate_error_log",\
&iniContext, false);
g_compress_old_access_log = iniGetBoolValue(NULL, "compress_old_access_log",
&iniContext, false);
g_compress_old_error_log = iniGetBoolValue(NULL, "compress_old_error_log",
&iniContext, false);
g_compress_error_log_days_before = iniGetIntValue(NULL,
"compress_error_log_days_before", &iniContext, 1);
g_compress_access_log_days_before = iniGetIntValue(NULL,
"compress_access_log_days_before", &iniContext, 1);
if (g_compress_old_error_log)
{
log_set_compress_log_flags(LOG_COMPRESS_FLAGS_ENABLED |
LOG_COMPRESS_FLAGS_NEW_THREAD);
log_set_compress_log_days_before(g_compress_error_log_days_before);
}
if (g_use_access_log && g_compress_old_access_log)
{
log_set_compress_log_flags_ex(&g_access_log_context,
LOG_COMPRESS_FLAGS_ENABLED |
LOG_COMPRESS_FLAGS_NEW_THREAD);
log_set_compress_log_days_before_ex(&g_access_log_context,
g_compress_access_log_days_before);
}
if ((result=get_time_item_from_conf(&iniContext, \
"error_log_rotate_time", &g_error_log_rotate_time, \
0, 0)) != 0)
@ -1785,6 +2098,16 @@ int storage_func_init(const char *filename, \
g_file_sync_skip_invalid_record = iniGetBoolValue(NULL, \
"file_sync_skip_invalid_record", &iniContext, false);
g_compress_binlog = iniGetBoolValue(NULL,
"compress_binlog", &iniContext, false);
if ((result=get_time_item_from_conf(&iniContext,
"compress_binlog_time", &g_compress_binlog_time, 1, 30)) != 0)
{
break;
}
g_check_store_path_mark = iniGetBoolValue(NULL,
"check_store_path_mark", &iniContext, true);
if ((result=fdfs_connection_pool_init(filename, &iniContext)) != 0)
{
break;
@ -1825,7 +2148,7 @@ int storage_func_init(const char *filename, \
"max_connections=%d, accept_threads=%d, " \
"work_threads=%d, " \
"disk_rw_separated=%d, disk_reader_threads=%d, " \
"disk_writer_threads=%d, " \
"disk_writer_threads=%d, disk_recovery_threads=%d, " \
"buff_size=%d KB, heart_beat_interval=%ds, " \
"stat_report_interval=%ds, tracker_server_count=%d, " \
"sync_wait_msec=%dms, sync_interval=%dms, " \
@ -1846,14 +2169,21 @@ int storage_func_init(const char *filename, \
"HTTP server port=%d, domain name=%s, " \
"use_access_log=%d, rotate_access_log=%d, " \
"access_log_rotate_time=%02d:%02d, " \
"compress_old_access_log=%d, " \
"compress_access_log_days_before=%d, " \
"rotate_error_log=%d, " \
"error_log_rotate_time=%02d:%02d, " \
"compress_old_error_log=%d, " \
"compress_error_log_days_before=%d, " \
"rotate_access_log_size=%"PRId64", " \
"rotate_error_log_size=%"PRId64", " \
"log_file_keep_days=%d, " \
"file_sync_skip_invalid_record=%d, " \
"use_connection_pool=%d, " \
"g_connection_pool_max_idle_time=%ds", \
"g_connection_pool_max_idle_time=%ds, " \
"compress_binlog=%d, " \
"compress_binlog_time=%02d:%02d, " \
"check_store_path_mark=%d", \
g_fdfs_version.major, g_fdfs_version.minor, \
g_fdfs_base_path, g_fdfs_store_paths.count, \
g_subdir_count_per_path, \
@ -1863,7 +2193,7 @@ int storage_func_init(const char *filename, \
g_client_bind_addr, g_max_connections, \
g_accept_threads, g_work_threads, g_disk_rw_separated, \
g_disk_reader_threads, g_disk_writer_threads, \
g_buff_size / 1024, \
g_disk_recovery_threads, g_buff_size / 1024, \
g_heart_beat_interval, g_stat_report_interval, \
g_tracker_group.server_count, g_sync_wait_usec / 1000, \
g_sync_interval / 1000, \
@ -1882,13 +2212,17 @@ int storage_func_init(const char *filename, \
g_key_namespace, g_keep_alive, \
g_http_port, g_http_domain, g_use_access_log, \
g_rotate_access_log, g_access_log_rotate_time.hour, \
g_access_log_rotate_time.minute, \
g_access_log_rotate_time.minute, g_compress_old_access_log, \
g_compress_access_log_days_before, \
g_rotate_error_log, g_error_log_rotate_time.hour, \
g_error_log_rotate_time.minute, \
g_error_log_rotate_time.minute, g_compress_old_error_log, \
g_compress_error_log_days_before, \
g_access_log_context.rotate_size, \
g_log_context.rotate_size, g_log_file_keep_days, \
g_file_sync_skip_invalid_record, \
g_use_connection_pool, g_connection_pool_max_idle_time);
g_use_connection_pool, g_connection_pool_max_idle_time, \
g_compress_binlog, g_compress_binlog_time.hour, \
g_compress_binlog_time.minute, g_check_store_path_mark);
#ifdef WITH_HTTPD
if (!g_http_params.disabled)
@ -1922,7 +2256,7 @@ int storage_func_init(const char *filename, \
return result;
}
if ((result=init_my_status_per_tracker()) != 0)
if ((result=init_my_result_per_tracker()) != 0)
{
return result;
}
@ -1968,6 +2302,11 @@ int storage_func_init(const char *filename, \
return result;
}
if ((result=storage_trunk_binlog_compress_check_recovery()) != 0)
{
return result;
}
if ((result=init_pthread_lock(&sync_stat_file_lock)) != 0)
{
return result;
@ -1986,10 +2325,10 @@ int storage_func_destroy()
{
for (i=0; i<g_fdfs_store_paths.count; i++)
{
if (g_fdfs_store_paths.paths[i] != NULL)
if (g_fdfs_store_paths.paths[i].path != NULL)
{
free(g_fdfs_store_paths.paths[i]);
g_fdfs_store_paths.paths[i] = NULL;
free(g_fdfs_store_paths.paths[i].path);
g_fdfs_store_paths.paths[i].path = NULL;
}
}
@ -2116,7 +2455,7 @@ int storage_set_tracker_client_ips(ConnectionInfo *conn,
for (i = 0; i < multi_ip.count; i++)
{
result = storage_insert_ip_addr_to_multi_ips(&g_tracker_client_ip,
multi_ip.ips[i], multi_ip.count);
multi_ip.ips[i].address, multi_ip.count);
if (result == 0)
{
if ((result=fdfs_check_and_format_ips(&g_tracker_client_ip,
@ -2127,12 +2466,12 @@ int storage_set_tracker_client_ips(ConnectionInfo *conn,
"my ip: %s not valid, error info: %s. "
"program exit!", __LINE__,
conn->ip_addr, conn->port,
multi_ip.ips[i], error_info);
multi_ip.ips[i].address, error_info);
return result;
}
insert_into_local_host_ip(multi_ip.ips[i]);
insert_into_local_host_ip(multi_ip.ips[i].address);
}
else if (result != EEXIST)
{
@ -2145,7 +2484,7 @@ int storage_set_tracker_client_ips(ConnectionInfo *conn,
"my ip: %s not consistent with client ips: %s "
"of other tracker client. program exit!", __LINE__,
conn->ip_addr, conn->port,
multi_ip.ips[i], ip_str);
multi_ip.ips[i].address, ip_str);
return result;
}
@ -2155,6 +2494,29 @@ int storage_set_tracker_client_ips(ConnectionInfo *conn,
return 0;
}
int storage_logic_to_local_full_filename(const char *logic_filename,
const int logic_filename_len, int *store_path_index,
char *full_filename, const int filename_size)
{
int result;
int filename_len;
char true_filename[128];
filename_len = logic_filename_len;
if ((result=storage_split_filename_ex(logic_filename,
&filename_len, true_filename, store_path_index)) != 0)
{
return result;
}
if ((result=fdfs_check_data_filename(true_filename, filename_len)) != 0)
{
return result;
}
snprintf(full_filename, filename_size, "%s/data/%s",
g_fdfs_store_paths.paths[*store_path_index].path, true_filename);
return 0;
}
/*
int write_serialized(int fd, const char *buff, size_t count, const bool bSync)

8
storage/storage_func.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_func.h
@ -37,6 +37,12 @@ bool storage_id_is_myself(const char *storage_id);
int storage_set_tracker_client_ips(ConnectionInfo *conn,
const int tracker_index);
int storage_check_and_make_global_data_path();
int storage_logic_to_local_full_filename(const char *logic_filename,
const int logic_filename_len, int *store_path_index,
char *full_filename, const int filename_size);
#define STORAGE_CHOWN(path, current_uid, current_gid) \
if (!(g_run_by_gid == current_gid && g_run_by_uid == current_uid)) \
{ \

20
storage/storage_global.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <netdb.h>
@ -15,7 +15,6 @@
#include "storage_global.h"
volatile bool g_continue_flag = true;
FDFSStorePathInfo *g_path_space_list = NULL;
int g_subdir_count_per_path = DEFAULT_DATA_DIR_COUNT_PER_PATH;
int g_server_port = FDFS_STORAGE_SERVER_DEF_PORT;
@ -32,6 +31,7 @@ bool g_disk_rw_direct = false;
bool g_disk_rw_separated = true;
int g_disk_reader_threads = DEFAULT_DISK_READER_THREADS;
int g_disk_writer_threads = DEFAULT_DISK_WRITER_THREADS;
int g_disk_recovery_threads = 1;
int g_extra_open_file_flags = 0;
int g_file_distribute_path_mode = FDFS_FILE_DIST_PATH_ROUND_ROBIN;
@ -80,6 +80,8 @@ in_addr_t g_server_id_in_filename = 0;
bool g_use_access_log = false; //if log to access log
bool g_rotate_access_log = false; //if rotate the access log every day
bool g_rotate_error_log = false; //if rotate the error log every day
bool g_compress_old_access_log = false; //if compress the old access log
bool g_compress_old_error_log = false; //if compress the old error log
bool g_use_storage_id = false; //identify storage by ID instead of IP address
byte g_id_type_in_filename = FDFS_ID_TYPE_IP_ADDRESS; //id type of the storage server in the filename
bool g_store_slave_file_use_link = false; //if store slave file use symbol link
@ -107,6 +109,10 @@ bool g_storage_ip_changed_auto_adjust = false;
bool g_thread_kill_done = false;
bool g_file_sync_skip_invalid_record = false;
bool g_check_store_path_mark = true;
bool g_compress_binlog = false;
TimeInfo g_compress_binlog_time = {0, 0};
int g_thread_stack_size = 512 * 1024;
int g_upload_priority = DEFAULT_UPLOAD_PRIORITY;
time_t g_up_time = 0;
@ -121,6 +127,8 @@ char g_exe_name[256] = {0};
#endif
int g_log_file_keep_days = 0;
int g_compress_access_log_days_before = 0;
int g_compress_error_log_days_before = 0;
struct storage_nio_thread_data *g_nio_thread_data = NULL;
struct storage_dio_thread_data *g_dio_thread_data = NULL;
@ -138,13 +146,14 @@ int storage_insert_ip_addr_to_multi_ips(FDFSMultiIP *multi_ip,
if (multi_ip->count == 0)
{
multi_ip->count = 1;
strcpy(multi_ip->ips[0], ip_addr);
multi_ip->ips[0].type = fdfs_get_ip_type(ip_addr);
strcpy(multi_ip->ips[0].address, ip_addr);
return 0;
}
for (i = 0; i < multi_ip->count; i++)
{
if (strcmp(multi_ip->ips[i], ip_addr) == 0)
if (strcmp(multi_ip->ips[i].address, ip_addr) == 0)
{
return EEXIST;
}
@ -155,7 +164,8 @@ int storage_insert_ip_addr_to_multi_ips(FDFSMultiIP *multi_ip,
return ENOSPC;
}
strcpy(multi_ip->ips[i], ip_addr);
multi_ip->ips[i].type = fdfs_get_ip_type(ip_addr);
strcpy(multi_ip->ips[i].address, ip_addr);
multi_ip->count++;
return 0;
}

26
storage/storage_global.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_global.h
@ -21,7 +21,6 @@
#include "client_global.h"
#include "fdht_types.h"
#include "fastcommon/local_ip_func.h"
#include "trunk_shared.h"
#ifdef WITH_HTTPD
#include "fdfs_http_shared.h"
@ -53,21 +52,15 @@ typedef struct
typedef struct
{
int total_mb; //total spaces
int free_mb; //free spaces
} FDFSStorePathInfo;
typedef struct
{
signed char my_status;
signed char src_storage_status;
signed char my_status; //my status from tracker server
signed char my_result; //my report result
signed char src_storage_result; //src storage report result
bool get_my_ip_done;
bool report_my_status;
} StorageStatusPerTracker;
extern volatile bool g_continue_flag;
extern FDFSStorePathInfo *g_path_space_list;
/* subdirs under store path, g_subdir_count * g_subdir_count 2 level subdirs */
extern int g_subdir_count_per_path;
@ -85,6 +78,7 @@ extern bool g_disk_rw_direct; //if file read / write directly
extern bool g_disk_rw_separated; //if disk read / write separated
extern int g_disk_reader_threads; //disk reader thread count per store base path
extern int g_disk_writer_threads; //disk writer thread count per store base path
extern int g_disk_recovery_threads; //disk recovery thread count
extern int g_extra_open_file_flags; //extra open file flags
extern int g_file_distribute_path_mode;
@ -135,6 +129,8 @@ extern byte g_id_type_in_filename; //id type of the storage server in the filena
extern bool g_use_access_log; //if log to access log
extern bool g_rotate_access_log; //if rotate the access log every day
extern bool g_rotate_error_log; //if rotate the error log every day
extern bool g_compress_old_access_log; //if compress the old access log
extern bool g_compress_old_error_log; //if compress the old error log
extern TimeInfo g_access_log_rotate_time; //rotate access log time base
extern TimeInfo g_error_log_rotate_time; //rotate error log time base
@ -162,6 +158,10 @@ extern bool g_thread_kill_done;
extern bool g_file_sync_skip_invalid_record;
extern bool g_check_store_path_mark;
extern bool g_compress_binlog;
extern TimeInfo g_compress_binlog_time; //compress binlog time base
extern int g_thread_stack_size;
extern int g_upload_priority;
extern time_t g_up_time;
@ -176,6 +176,8 @@ extern char g_exe_name[256];
#endif
extern int g_log_file_keep_days;
extern int g_compress_access_log_days_before;
extern int g_compress_error_log_days_before;
extern struct storage_nio_thread_data *g_nio_thread_data; //network io thread data
extern struct storage_dio_thread_data *g_dio_thread_data; //disk io thread data

29
storage/storage_ip_changed_dealer.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
@ -76,9 +76,9 @@ static int storage_report_ip_changed(ConnectionInfo *pTrackerServer)
pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_REPORT_IP_CHANGED;
strcpy(out_buff + sizeof(TrackerHeader), g_group_name);
strcpy(out_buff + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN, \
g_last_storage_ip.ips[0]);
g_last_storage_ip.ips[0].address);
strcpy(out_buff + sizeof(TrackerHeader) + FDFS_GROUP_NAME_MAX_LEN + \
IP_ADDRESS_SIZE, g_tracker_client_ip.ips[0]);
IP_ADDRESS_SIZE, g_tracker_client_ip.ips[0].address);
if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff, \
sizeof(out_buff), g_fdfs_network_timeout)) != 0)
@ -92,10 +92,11 @@ static int storage_report_ip_changed(ConnectionInfo *pTrackerServer)
}
pInBuff = in_buff;
result = fdfs_recv_response(pTrackerServer, \
result = fdfs_recv_response(pTrackerServer,
&pInBuff, 0, &in_bytes);
if (result == 0 || result == EALREADY || result == ENOENT)
if (result == 0 || result == EALREADY || result == ENOENT
|| result == EEXIST)
{
if (result != 0)
{
@ -107,11 +108,11 @@ static int storage_report_ip_changed(ConnectionInfo *pTrackerServer)
}
else
{
logError("file: "__FILE__", line: %d, " \
"tracker server %s:%d, recv data fail or " \
"response status != 0, " \
"errno: %d, error info: %s", \
__LINE__, pTrackerServer->ip_addr, \
logError("file: "__FILE__", line: %d, "
"tracker server %s:%d, recv data fail or "
"response status != 0, "
"errno: %d, error info: %s",
__LINE__, pTrackerServer->ip_addr,
pTrackerServer->port, result, STRERROR(result));
return result == EBUSY ? 0 : result;
}
@ -208,15 +209,15 @@ static int storage_report_storage_ip_addr()
logDebug("file: "__FILE__", line: %d, "
"last my ip is %s, current my ip is %s",
__LINE__, g_last_storage_ip.ips[0],
g_tracker_client_ip.ips[0]);
__LINE__, g_last_storage_ip.ips[0].address,
g_tracker_client_ip.ips[0].address);
if (g_last_storage_ip.count == 0)
{
return storage_write_to_sync_ini_file();
}
else if (strcmp(g_tracker_client_ip.ips[0],
g_last_storage_ip.ips[0]) == 0)
else if (strcmp(g_tracker_client_ip.ips[0].address,
g_last_storage_ip.ips[0].address) == 0)
{
return 0;
}

2
storage/storage_ip_changed_dealer.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_ip_changed_dealer.h

54
storage/storage_nio.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
#include <stdio.h>
@ -38,13 +38,6 @@ static void client_sock_read(int sock, short event, void *arg);
static void client_sock_write(int sock, short event, void *arg);
static int storage_nio_init(struct fast_task_info *pTask);
void add_to_deleted_list(struct fast_task_info *pTask)
{
((StorageClientInfo *)pTask->arg)->canceled = true;
pTask->next = pTask->thread_data->deleted_list;
pTask->thread_data->deleted_list = pTask;
}
void task_finish_clean_up(struct fast_task_info *pTask)
{
StorageClientInfo *pClientInfo;
@ -66,6 +59,7 @@ void task_finish_clean_up(struct fast_task_info *pTask)
pTask->event.timer.expires = 0;
}
pTask->canceled = false;
memset(pTask->arg, 0, sizeof(StorageClientInfo));
free_queue_push(pTask);
@ -87,7 +81,7 @@ static int set_recv_event(struct fast_task_info *pTask)
pTask->event.fd, IOEVENT_READ, pTask) != 0)
{
result = errno != 0 ? errno : ENOENT;
add_to_deleted_list(pTask);
ioevent_add_to_deleted_list(pTask);
logError("file: "__FILE__", line: %d, "\
"ioevent_modify fail, " \
@ -112,7 +106,7 @@ static int set_send_event(struct fast_task_info *pTask)
pTask->event.fd, IOEVENT_WRITE, pTask) != 0)
{
result = errno != 0 ? errno : ENOENT;
add_to_deleted_list(pTask);
ioevent_add_to_deleted_list(pTask);
logError("file: "__FILE__", line: %d, "\
"ioevent_modify fail, " \
@ -210,7 +204,7 @@ void storage_recv_notify_read(int sock, short event, void *arg)
if (result != 0)
{
add_to_deleted_list(pTask);
ioevent_add_to_deleted_list(pTask);
}
}
}
@ -248,7 +242,7 @@ static void client_sock_read(int sock, short event, void *arg)
pTask = (struct fast_task_info *)arg;
pClientInfo = (StorageClientInfo *)pTask->arg;
if (pClientInfo->canceled)
if (pTask->canceled)
{
return;
}
@ -267,7 +261,9 @@ static void client_sock_read(int sock, short event, void *arg)
if (event & IOEVENT_TIMEOUT)
{
if (pClientInfo->total_offset == 0 && pTask->req_count > 0)
if (pClientInfo->total_offset == 0)
{
if (pTask->req_count > 0)
{
pTask->event.timer.expires = g_current_time +
g_fdfs_network_timeout;
@ -276,12 +272,22 @@ static void client_sock_read(int sock, short event, void *arg)
}
else
{
logError("file: "__FILE__", line: %d, " \
"client ip: %s, recv timeout, " \
"recv offset: %d, expect length: %d", \
__LINE__, pTask->client_ip, \
pTask->offset, pTask->length);
logWarning("file: "__FILE__", line: %d, "
"client ip: %s, recv timeout. "
"after the connection is established, "
"you must send a request before %ds timeout, "
"maybe connections leak in you application.",
__LINE__, pTask->client_ip, g_fdfs_network_timeout);
task_finish_clean_up(pTask);
}
}
else
{
logError("file: "__FILE__", line: %d, "
"client ip: %s, recv timeout, "
"recv offset: %d, expect length: %d, "
"req_count: %"PRId64, __LINE__, pTask->client_ip,
pTask->offset, pTask->length, pTask->req_count);
task_finish_clean_up(pTask);
}
@ -425,15 +431,17 @@ static void client_sock_write(int sock, short event, void *arg)
pTask = (struct fast_task_info *)arg;
pClientInfo = (StorageClientInfo *)pTask->arg;
if (pClientInfo->canceled)
if (pTask->canceled)
{
return;
}
if (event & IOEVENT_TIMEOUT)
{
logError("file: "__FILE__", line: %d, " \
"send timeout", __LINE__);
logError("file: "__FILE__", line: %d, "
"client ip: %s, send timeout, offset: %d, "
"remain bytes: %d", __LINE__, pTask->client_ip,
pTask->offset, pTask->length - pTask->offset);
task_finish_clean_up(pTask);
return;
@ -441,7 +449,7 @@ static void client_sock_write(int sock, short event, void *arg)
if (event & IOEVENT_ERROR)
{
logDebug("file: "__FILE__", line: %d, " \
logDebug("file: "__FILE__", line: %d, "
"client ip: %s, recv error event: %d, "
"close connection", __LINE__, pTask->client_ip, event);

7
storage/storage_nio.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//tracker_nio.h
@ -46,6 +46,8 @@ typedef void (*DeleteFileLogCallback)(struct fast_task_info *pTask, \
typedef void (*FileDealDoneCallback)(struct fast_task_info *pTask, \
const int err_no);
typedef int (*FileDealContinueCallback)(struct fast_task_info *pTask);
typedef int (*FileBeforeOpenCallback)(struct fast_task_info *pTask);
typedef int (*FileBeforeCloseCallback)(struct fast_task_info *pTask);
@ -109,6 +111,7 @@ typedef struct
int64_t start; //the start offset of file
int64_t end; //the end offset of file
int64_t offset; //the current offset of file
FileDealContinueCallback continue_callback;
FileDealDoneCallback done_callback;
DeleteFileLogCallback log_callback;
@ -118,7 +121,6 @@ typedef struct
typedef struct
{
int nio_thread_index; //nio thread index
bool canceled;
char stage; //nio stage, send or recv
char storage_server_id[FDFS_STORAGE_ID_MAX_SIZE];
@ -149,7 +151,6 @@ void storage_recv_notify_read(int sock, short event, void *arg);
int storage_send_add_event(struct fast_task_info *pTask);
void task_finish_clean_up(struct fast_task_info *pTask);
void add_to_deleted_list(struct fast_task_info *pTask);
#ifdef __cplusplus
}

155
storage/storage_param_getter.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
@ -74,30 +74,30 @@ int storage_get_params_from_tracker()
char reserved_space_str[32];
char *pIdType;
if ((result=fdfs_get_ini_context_from_tracker(&g_tracker_group, \
&iniContext, (bool * volatile)&g_continue_flag, \
if ((result=fdfs_get_ini_context_from_tracker(&g_tracker_group,
&iniContext, (bool * volatile)&g_continue_flag,
g_client_bind_addr, g_bind_addr)) != 0)
{
return result;
}
g_storage_ip_changed_auto_adjust = iniGetBoolValue(NULL, \
"storage_ip_changed_auto_adjust", \
g_storage_ip_changed_auto_adjust = iniGetBoolValue(NULL,
"storage_ip_changed_auto_adjust",
&iniContext, false);
g_store_path_mode = iniGetIntValue(NULL, "store_path", &iniContext, \
g_store_path_mode = iniGetIntValue(NULL, "store_path", &iniContext,
FDFS_STORE_PATH_ROUND_ROBIN);
if ((result=fdfs_parse_storage_reserved_space(&iniContext, \
if ((result=fdfs_parse_storage_reserved_space(&iniContext,
&g_storage_reserved_space)) != 0)
{
iniFreeContext(&iniContext);
return result;
}
if (g_storage_reserved_space.flag == \
if (g_storage_reserved_space.flag ==
TRACKER_STORAGE_RESERVED_SPACE_FLAG_MB)
{
g_avg_storage_reserved_mb = g_storage_reserved_space.rs.mb \
g_avg_storage_reserved_mb = g_storage_reserved_space.rs.mb
/ g_fdfs_store_paths.count;
}
else
@ -105,41 +105,61 @@ int storage_get_params_from_tracker()
g_avg_storage_reserved_mb = 0;
}
g_use_storage_id = iniGetBoolValue(NULL, "use_storage_id", \
g_use_storage_id = iniGetBoolValue(NULL, "use_storage_id",
&iniContext, false);
use_trunk_file = iniGetBoolValue(NULL, "use_trunk_file", \
use_trunk_file = iniGetBoolValue(NULL, "use_trunk_file",
&iniContext, false);
g_slot_min_size = iniGetIntValue(NULL, "slot_min_size", \
g_slot_min_size = iniGetIntValue(NULL, "slot_min_size",
&iniContext, 256);
g_trunk_file_size = iniGetIntValue(NULL, "trunk_file_size", \
g_trunk_file_size = iniGetIntValue(NULL, "trunk_file_size",
&iniContext, 64 * 1024 * 1024);
g_slot_max_size = iniGetIntValue(NULL, "slot_max_size", \
&iniContext, g_trunk_file_size / 2);
g_slot_max_size = iniGetIntValue(NULL, "slot_max_size",
&iniContext, g_trunk_file_size / 4);
g_trunk_alloc_alignment_size = iniGetIntValue(NULL,
"trunk_alloc_alignment_size", &iniContext, 0);
if (g_slot_min_size < g_trunk_alloc_alignment_size)
{
g_slot_min_size = g_trunk_alloc_alignment_size;
}
g_trunk_create_file_advance = iniGetBoolValue(NULL, \
g_trunk_create_file_advance = iniGetBoolValue(NULL,
"trunk_create_file_advance", &iniContext, false);
if ((result=get_time_item_from_conf(&iniContext, \
"trunk_create_file_time_base", \
if ((result=get_time_item_from_conf(&iniContext,
"trunk_create_file_time_base",
&g_trunk_create_file_time_base, 2, 0)) != 0)
{
iniFreeContext(&iniContext);
return result;
}
g_trunk_create_file_interval = iniGetIntValue(NULL, \
"trunk_create_file_interval", &iniContext, \
g_trunk_create_file_interval = iniGetIntValue(NULL,
"trunk_create_file_interval", &iniContext,
86400);
g_trunk_create_file_space_threshold = iniGetInt64Value(NULL, \
"trunk_create_file_space_threshold", \
g_trunk_create_file_space_threshold = iniGetInt64Value(NULL,
"trunk_create_file_space_threshold",
&iniContext, 0);
g_trunk_init_check_occupying = iniGetBoolValue(NULL, \
g_trunk_init_check_occupying = iniGetBoolValue(NULL,
"trunk_init_check_occupying", &iniContext, false);
g_trunk_init_reload_from_binlog = iniGetBoolValue(NULL, \
g_trunk_init_reload_from_binlog = iniGetBoolValue(NULL,
"trunk_init_reload_from_binlog", &iniContext, false);
g_trunk_compress_binlog_min_interval = iniGetIntValue(NULL, \
g_trunk_free_space_merge = iniGetBoolValue(NULL,
"trunk_free_space_merge", &iniContext, false);
g_delete_unused_trunk_files = iniGetBoolValue(NULL,
"delete_unused_trunk_files", &iniContext, false);
g_trunk_compress_binlog_min_interval = iniGetIntValue(NULL,
"trunk_compress_binlog_min_interval", &iniContext, 0);
g_trunk_compress_binlog_interval = iniGetIntValue(NULL,
"trunk_compress_binlog_interval", &iniContext, 0);
if ((result=get_time_item_from_conf(&iniContext,
"trunk_compress_binlog_time_base",
&g_trunk_compress_binlog_time_base, 3, 0)) != 0)
{
return result;
}
g_store_slave_file_use_link = iniGetBoolValue(NULL, \
g_trunk_binlog_max_backups = iniGetIntValue(NULL,
"trunk_binlog_max_backups", &iniContext, 0);
g_store_slave_file_use_link = iniGetBoolValue(NULL,
"store_slave_file_use_link", &iniContext, false);
pIdType = iniGetStrValue(NULL, "id_type_in_filename", &iniContext);
@ -170,43 +190,56 @@ int storage_get_params_from_tracker()
}
g_if_use_trunk_file = use_trunk_file;
logInfo("file: "__FILE__", line: %d, " \
"use_storage_id=%d, " \
"id_type_in_filename=%s, " \
"storage_ip_changed_auto_adjust=%d, " \
"store_path=%d, " \
"reserved_storage_space=%s, " \
"use_trunk_file=%d, " \
"slot_min_size=%d, " \
"slot_max_size=%d MB, " \
"trunk_file_size=%d MB, " \
"trunk_create_file_advance=%d, " \
"trunk_create_file_time_base=%02d:%02d, " \
"trunk_create_file_interval=%d, " \
"trunk_create_file_space_threshold=%d GB, " \
"trunk_init_check_occupying=%d, " \
"trunk_init_reload_from_binlog=%d, " \
"trunk_compress_binlog_min_interval=%d, " \
"store_slave_file_use_link=%d", \
__LINE__, g_use_storage_id, \
g_id_type_in_filename == FDFS_ID_TYPE_SERVER_ID ? "id" : "ip", \
g_storage_ip_changed_auto_adjust, \
g_store_path_mode, fdfs_storage_reserved_space_to_string( \
&g_storage_reserved_space, reserved_space_str), \
g_if_use_trunk_file, g_slot_min_size, \
g_slot_max_size / FDFS_ONE_MB, \
g_trunk_file_size / FDFS_ONE_MB, \
g_trunk_create_file_advance, \
g_trunk_create_file_time_base.hour, \
g_trunk_create_file_time_base.minute, \
g_trunk_create_file_interval, \
(int)(g_trunk_create_file_space_threshold / \
(FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying, \
g_trunk_init_reload_from_binlog, \
g_trunk_compress_binlog_min_interval, \
logInfo("file: "__FILE__", line: %d, "
"use_storage_id=%d, "
"id_type_in_filename=%s, "
"storage_ip_changed_auto_adjust=%d, "
"store_path=%d, "
"reserved_storage_space=%s, "
"use_trunk_file=%d, "
"slot_min_size=%d, "
"slot_max_size=%d KB, "
"trunk_alloc_alignment_size=%d, "
"trunk_file_size=%d MB, "
"trunk_create_file_advance=%d, "
"trunk_create_file_time_base=%02d:%02d, "
"trunk_create_file_interval=%d, "
"trunk_create_file_space_threshold=%d GB, "
"trunk_init_check_occupying=%d, "
"trunk_init_reload_from_binlog=%d, "
"trunk_free_space_merge=%d, "
"delete_unused_trunk_files=%d, "
"trunk_compress_binlog_min_interval=%d, "
"trunk_compress_binlog_interval=%d, "
"trunk_compress_binlog_time_base=%02d:%02d, "
"trunk_binlog_max_backups=%d, "
"store_slave_file_use_link=%d",
__LINE__, g_use_storage_id,
g_id_type_in_filename == FDFS_ID_TYPE_SERVER_ID ? "id" : "ip",
g_storage_ip_changed_auto_adjust,
g_store_path_mode, fdfs_storage_reserved_space_to_string(
&g_storage_reserved_space, reserved_space_str),
g_if_use_trunk_file, g_slot_min_size,
g_slot_max_size / 1024,
g_trunk_alloc_alignment_size,
g_trunk_file_size / FDFS_ONE_MB,
g_trunk_create_file_advance,
g_trunk_create_file_time_base.hour,
g_trunk_create_file_time_base.minute,
g_trunk_create_file_interval,
(int)(g_trunk_create_file_space_threshold /
(FDFS_ONE_MB * 1024)), g_trunk_init_check_occupying,
g_trunk_init_reload_from_binlog,
g_trunk_free_space_merge,
g_delete_unused_trunk_files,
g_trunk_compress_binlog_min_interval,
g_trunk_compress_binlog_interval,
g_trunk_compress_binlog_time_base.hour,
g_trunk_compress_binlog_time_base.minute,
g_trunk_binlog_max_backups,
g_store_slave_file_use_link);
if (g_use_storage_id && *g_sync_src_id != '\0' && \
if (g_use_storage_id && *g_sync_src_id != '\0' &&
!fdfs_is_server_id_valid(g_sync_src_id))
{
if ((result=storage_convert_src_server_id()) == 0)

2
storage/storage_param_getter.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_param_getter.h

1133
storage/storage_service.c
View File

File diff suppressed because it is too large Load Diff

4
storage/storage_service.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_service.h
@ -36,7 +36,7 @@ void storage_service_destroy();
int fdfs_stat_file_sync_func(void *args);
int storage_deal_task(struct fast_task_info *pTask);
void storage_nio_notify(struct fast_task_info *pTask);
int storage_nio_notify(struct fast_task_info *pTask);
void storage_accept_loop(int server_sock);
int storage_terminate_threads();

850
storage/storage_sync.c
View File

File diff suppressed because it is too large Load Diff

18
storage/storage_sync.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_sync.h
@ -11,6 +11,7 @@
#ifndef _STORAGE_SYNC_H_
#define _STORAGE_SYNC_H_
#include "fastcommon/fc_list.h"
#include "storage_func.h"
#define STORAGE_OP_TYPE_SOURCE_CREATE_FILE 'C' //upload file
@ -20,6 +21,7 @@
#define STORAGE_OP_TYPE_SOURCE_MODIFY_FILE 'M' //for part modify
#define STORAGE_OP_TYPE_SOURCE_TRUNCATE_FILE 'T' //truncate file
#define STORAGE_OP_TYPE_SOURCE_CREATE_LINK 'L' //create symbol link
#define STORAGE_OP_TYPE_SOURCE_RENAME_FILE 'R' //rename appender file to normal file
#define STORAGE_OP_TYPE_REPLICA_CREATE_FILE 'c'
#define STORAGE_OP_TYPE_REPLICA_APPEND_FILE 'a'
#define STORAGE_OP_TYPE_REPLICA_DELETE_FILE 'd'
@ -27,6 +29,7 @@
#define STORAGE_OP_TYPE_REPLICA_MODIFY_FILE 'm'
#define STORAGE_OP_TYPE_REPLICA_TRUNCATE_FILE 't'
#define STORAGE_OP_TYPE_REPLICA_CREATE_LINK 'l'
#define STORAGE_OP_TYPE_REPLICA_RENAME_FILE 'r'
#define STORAGE_BINLOG_BUFFER_SIZE 64 * 1024
#define STORAGE_BINLOG_LINE_SIZE 256
@ -37,13 +40,14 @@ extern "C" {
typedef struct
{
struct fc_list_head link;
char storage_id[FDFS_STORAGE_ID_MAX_SIZE];
char mark_filename[MAX_PATH_SIZE];
bool need_sync_old;
bool sync_old_done;
bool last_file_exist; //if the last file exist on the dest server
BinLogBuffer binlog_buff;
time_t until_timestamp;
int mark_fd;
int binlog_index;
int binlog_fd;
int64_t binlog_offset;
@ -88,7 +92,7 @@ int storage_sync_thread_start(const FDFSStorageBrief *pStorage);
int kill_storage_sync_threads();
int fdfs_binlog_sync_func(void *args);
char *get_mark_filename_by_reader(const void *pArg, char *full_filename);
char *get_mark_filename_by_reader(StorageBinLogReader *pReader);
int storage_unlink_mark_file(const char *storage_id);
int storage_rename_mark_file(const char *old_ip_addr, const int old_port, \
const char *new_ip_addr, const int new_port);
@ -99,9 +103,15 @@ int storage_open_readable_binlog(StorageBinLogReader *pReader, \
int storage_reader_init(FDFSStorageBrief *pStorage, StorageBinLogReader *pReader);
void storage_reader_destroy(StorageBinLogReader *pReader);
int storage_report_storage_status(const char *storage_id, \
int storage_report_storage_status(const char *storage_id,
const char *ip_addr, const char status);
int fdfs_binlog_compress_func(void *args);
void storage_reader_add_to_list(StorageBinLogReader *pReader);
void storage_reader_remove_from_list(StorageBinLogReader *pReader);
#ifdef __cplusplus
}
#endif

15
storage/storage_sync_func.c
View File

@ -2,7 +2,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_sync_func.c
@ -29,7 +29,7 @@
#include "storage_func.h"
#include "storage_sync_func.h"
void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage,
void storage_sync_connect_storage_server_ex(const FDFSStorageBrief *pStorage,
ConnectionInfo *conn, bool *check_flag)
{
int nContinuousFail;
@ -67,7 +67,8 @@ void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage,
{
ip_addrs.count = 1;
ip_addrs.index = 0;
strcpy(ip_addrs.ips[0], pStorage->ip_addr);
ip_addrs.ips[0].type = fdfs_get_ip_type(pStorage->ip_addr);
strcpy(ip_addrs.ips[0].address, pStorage->ip_addr);
}
conn->sock = -1;
@ -81,10 +82,10 @@ void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage,
{
for (i=0; i<ip_addrs.count; i++)
{
strcpy(conn->ip_addr, ip_addrs.ips[i]);
strcpy(conn->ip_addr, ip_addrs.ips[i].address);
conn->sock = socketCreateExAuto(conn->ip_addr,
g_fdfs_connect_timeout, O_NONBLOCK,
g_client_bind_addr ? g_bind_addr : NULL, &result);
O_NONBLOCK, g_client_bind_addr ?
g_bind_addr : NULL, &result);
if (conn->sock < 0)
{
logCrit("file: "__FILE__", line: %d, "
@ -148,7 +149,7 @@ void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage,
logError("file: "__FILE__", line: %d, "
"connect to storage server %s:%d fail, "
"try count: %d, errno: %d, error info: %s",
__LINE__, ip_addrs.ips[i], g_server_port, avg_fails,
__LINE__, ip_addrs.ips[i].address, g_server_port, avg_fails,
conn_results[i], STRERROR(conn_results[i]));
}
}

6
storage/storage_sync_func.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//storage_sync_func.h
@ -17,11 +17,11 @@
extern "C" {
#endif
void storage_sync_connect_storage_server_ex(FDFSStorageBrief *pStorage,
void storage_sync_connect_storage_server_ex(const FDFSStorageBrief *pStorage,
ConnectionInfo *conn, bool *check_flag);
static inline void storage_sync_connect_storage_server(
FDFSStorageBrief *pStorage, ConnectionInfo *conn)
const FDFSStorageBrief *pStorage, ConnectionInfo *conn)
{
bool check_flag = true;
storage_sync_connect_storage_server_ex(pStorage,

413
storage/tracker_client_thread.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
@ -38,6 +38,7 @@
#include "storage_param_getter.h"
#define TRUNK_FILE_CREATOR_TASK_ID 88
#define TRUNK_BINLOG_COMPRESS_TASK_ID 89
static pthread_mutex_t reporter_thread_lock;
@ -45,12 +46,16 @@ static pthread_mutex_t reporter_thread_lock;
static pthread_t *report_tids = NULL;
static bool need_rejoin_tracker = false;
static int tracker_heart_beat(ConnectionInfo *pTrackerServer, \
int *pstat_chg_sync_count, bool *bServerPortChanged);
static int tracker_report_df_stat(ConnectionInfo *pTrackerServer, \
bool *bServerPortChanged);
static int tracker_report_sync_timestamp(ConnectionInfo *pTrackerServer, \
static int tracker_heart_beat(ConnectionInfo *pTrackerServer,
const int tracker_index, int *pstat_chg_sync_count,
bool *bServerPortChanged);
static int tracker_report_df_stat(ConnectionInfo *pTrackerServer,
const int tracker_index, bool *bServerPortChanged);
static int tracker_report_sync_timestamp(ConnectionInfo *pTrackerServer,
const int tracker_index, bool *bServerPortChanged);
static int tracker_storage_change_status(ConnectionInfo *pTrackerServer,
const int tracker_index);
static int tracker_sync_dest_req(ConnectionInfo *pTrackerServer);
static int tracker_sync_dest_query(ConnectionInfo *pTrackerServer);
@ -390,14 +395,14 @@ static void *tracker_report_thread_entrance(void *arg)
sync_old_done = true;
}
g_my_report_status[tracker_index].src_storage_status =
g_my_report_status[tracker_index].src_storage_result =
tracker_sync_notify(conn, tracker_index);
if (g_my_report_status[tracker_index].src_storage_status != 0)
if (g_my_report_status[tracker_index].src_storage_result != 0)
{
int k;
for (k=0; k<g_tracker_group.server_count; k++)
{
if (g_my_report_status[k].src_storage_status != ENOENT)
if (g_my_report_status[k].src_storage_result != ENOENT)
{
break;
}
@ -406,22 +411,20 @@ static void *tracker_report_thread_entrance(void *arg)
if (k == g_tracker_group.server_count)
{ //src storage server already be deleted
int my_status;
if (tracker_get_storage_max_status( \
&g_tracker_group, g_group_name, \
tracker_client_ip, my_server_id, \
if (tracker_get_storage_max_status(
&g_tracker_group, g_group_name,
tracker_client_ip, my_server_id,
&my_status) == 0)
{
tracker_sync_dest_query(conn);
if (my_status < FDFS_STORAGE_STATUS_OFFLINE
&& g_sync_old_done)
{ //need re-sync old files
pthread_mutex_lock( \
&reporter_thread_lock);
pthread_mutex_lock(&reporter_thread_lock);
g_sync_old_done = false;
sync_old_done = g_sync_old_done;
storage_write_to_sync_ini_file();
pthread_mutex_unlock( \
&reporter_thread_lock);
pthread_mutex_unlock(&reporter_thread_lock);
}
}
}
@ -442,10 +445,11 @@ static void *tracker_report_thread_entrance(void *arg)
while (g_continue_flag)
{
current_time = g_current_time;
if (current_time - last_beat_time >= \
if (current_time - last_beat_time >=
g_heart_beat_interval)
{
if (tracker_heart_beat(conn, &stat_chg_sync_count,
if (tracker_heart_beat(conn, tracker_index,
&stat_chg_sync_count,
&bServerPortChanged) != 0)
{
break;
@ -464,8 +468,9 @@ static void *tracker_report_thread_entrance(void *arg)
current_time - last_sync_report_time >=
g_heart_beat_interval)
{
if (tracker_report_sync_timestamp(
conn, &bServerPortChanged)!=0)
if (tracker_report_sync_timestamp(conn,
tracker_index,
&bServerPortChanged) != 0)
{
break;
}
@ -478,6 +483,7 @@ static void *tracker_report_thread_entrance(void *arg)
g_stat_report_interval)
{
if (tracker_report_df_stat(conn,
tracker_index,
&bServerPortChanged) != 0)
{
break;
@ -486,6 +492,16 @@ static void *tracker_report_thread_entrance(void *arg)
last_df_report_time = current_time;
}
if (g_my_report_status[tracker_index].report_my_status)
{
if (tracker_storage_change_status(conn, tracker_index) == 0)
{
g_my_report_status[tracker_index].report_my_status = false;
}
break;
}
if (g_if_trunker_self)
{
if (last_trunk_file_id < g_current_trunk_file_id)
@ -717,8 +733,48 @@ static int tracker_start_sync_threads(const FDFSStorageBrief *pStorage)
return result;
}
static int tracker_merge_servers(ConnectionInfo *pTrackerServer, \
FDFSStorageBrief *briefServers, const int server_count)
static void tracker_check_my_status(const int tracker_index)
{
int my_status;
int leader_index;
int leader_status;
leader_index = g_tracker_group.leader_index;
if ((leader_index < 0) || (tracker_index == leader_index))
{
return;
}
my_status = g_my_report_status[tracker_index].my_status;
leader_status = g_my_report_status[leader_index].my_status;
if (my_status < 0 || leader_status < 0) //NOT inited
{
return;
}
if (my_status == leader_status)
{
return;
}
if (FDFS_IS_AVAILABLE_STATUS(my_status) &&
FDFS_IS_AVAILABLE_STATUS(leader_status))
{
return;
}
g_my_report_status[tracker_index].report_my_status = true;
logInfo("file: "__FILE__", line: %d, "
"my status: %d (%s) from tracker #%d != my status: %d (%s) "
"from leader tracker #%d, set report_my_status to true",
__LINE__, my_status, get_storage_status_caption(
my_status), tracker_index, leader_status,
get_storage_status_caption(leader_status), leader_index);
}
static int tracker_merge_servers(ConnectionInfo *pTrackerServer,
const int tracker_index, FDFSStorageBrief *briefServers,
const int server_count)
{
FDFSStorageBrief *pServer;
FDFSStorageBrief *pEnd;
@ -744,8 +800,14 @@ static int tracker_merge_servers(ConnectionInfo *pTrackerServer, \
{
memcpy(&(targetServer.server),pServer,sizeof(FDFSStorageBrief));
ppFound = (FDFSStorageServer **)bsearch(&pTargetServer, \
g_sorted_storages, g_storage_count, \
if (strcmp(pServer->id, g_my_server_id_str) == 0)
{
g_my_report_status[tracker_index].my_status = pServer->status;
tracker_check_my_status(tracker_index);
}
ppFound = (FDFSStorageServer **)bsearch(&pTargetServer,
g_sorted_storages, g_storage_count,
sizeof(FDFSStorageServer *), storage_cmp_by_server_id);
if (ppFound != NULL)
{
@ -815,9 +877,11 @@ static int tracker_merge_servers(ConnectionInfo *pTrackerServer, \
FDFS_STORAGE_STATUS_SYNCING)) && \
((*ppFound)->server.status > pServer->status))
{
pServer->id[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0';
*(pServer->ip_addr + IP_ADDRESS_SIZE - 1) = '\0';
if (is_local_host_ip(pServer->ip_addr) && \
buff2int(pServer->port) == g_server_port)
if ((strcmp(pServer->id, g_my_server_id_str) == 0) ||
(is_local_host_ip(pServer->ip_addr) &&
buff2int(pServer->port) == g_server_port))
{
need_rejoin_tracker = true;
logWarning("file: "__FILE__", line: %d, " \
@ -1031,6 +1095,21 @@ static int notify_reselect_tracker_leader(TrackerServerInfo *pTrackerServer)
return result;
}
static void check_my_status_for_all_trackers()
{
int tracker_index;
if (g_tracker_group.leader_index < 0)
{
return;
}
for (tracker_index=0; tracker_index<g_tracker_group.server_count;
tracker_index++)
{
tracker_check_my_status(tracker_index);
}
}
static void set_tracker_leader(const int leader_index)
{
int old_index;
@ -1064,7 +1143,12 @@ static void set_tracker_leader(const int leader_index)
}
}
}
if (g_tracker_group.leader_index != leader_index)
{
g_tracker_group.leader_index = leader_index;
check_my_status_for_all_trackers();
}
}
static void get_tracker_leader()
@ -1082,6 +1166,8 @@ static void get_tracker_leader()
if (tracker_status.if_leader)
{
g_tracker_group.leader_index = i;
check_my_status_for_all_trackers();
logInfo("file: "__FILE__", line: %d, "
"the tracker server leader is #%d. %s:%d",
__LINE__, i, tracker_server.connections[0].ip_addr,
@ -1121,8 +1207,76 @@ static void set_trunk_server(const char *ip_addr, const int port)
}
}
static int tracker_check_response(ConnectionInfo *pTrackerServer, \
bool *bServerPortChanged)
static int do_set_trunk_server_myself(ConnectionInfo *pTrackerServer)
{
int result;
ScheduleArray scheduleArray;
ScheduleEntry entries[2];
ScheduleEntry *entry;
tracker_fetch_trunk_fid(pTrackerServer);
g_if_trunker_self = true;
if ((result=storage_trunk_init()) != 0)
{
return result;
}
scheduleArray.entries = entries;
entry = entries;
if (g_trunk_create_file_advance &&
g_trunk_create_file_interval > 0)
{
INIT_SCHEDULE_ENTRY_EX(*entry, TRUNK_FILE_CREATOR_TASK_ID,
g_trunk_create_file_time_base,
g_trunk_create_file_interval,
trunk_create_trunk_file_advance, NULL);
entry->new_thread = true;
entry++;
}
if (g_trunk_compress_binlog_interval > 0)
{
INIT_SCHEDULE_ENTRY_EX(*entry, TRUNK_BINLOG_COMPRESS_TASK_ID,
g_trunk_compress_binlog_time_base,
g_trunk_compress_binlog_interval,
trunk_binlog_compress_func, NULL);
entry->new_thread = true;
entry++;
}
scheduleArray.count = entry - entries;
if (scheduleArray.count > 0)
{
sched_add_entries(&scheduleArray);
}
trunk_sync_thread_start_all();
return 0;
}
static void do_unset_trunk_server_myself(ConnectionInfo *pTrackerServer)
{
tracker_report_trunk_fid(pTrackerServer);
g_if_trunker_self = false;
trunk_waiting_sync_thread_exit();
storage_trunk_destroy_ex(true, true);
if (g_trunk_create_file_advance &&
g_trunk_create_file_interval > 0)
{
sched_del_entry(TRUNK_FILE_CREATOR_TASK_ID);
}
if (g_trunk_compress_binlog_interval > 0)
{
sched_del_entry(TRUNK_BINLOG_COMPRESS_TASK_ID);
}
}
static int tracker_check_response(ConnectionInfo *pTrackerServer,
const int tracker_index, bool *bServerPortChanged)
{
int64_t nInPackLen;
TrackerHeader resp;
@ -1301,11 +1455,13 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \
{
int port;
pBriefServers->id[FDFS_STORAGE_ID_MAX_SIZE - 1] = '\0';
pBriefServers->ip_addr[IP_ADDRESS_SIZE - 1] = '\0';
port = buff2int(pBriefServers->port);
set_trunk_server(pBriefServers->ip_addr, port);
if (is_local_host_ip(pBriefServers->ip_addr) &&
port == g_server_port)
if ((strcmp(pBriefServers->id, g_my_server_id_str) == 0) ||
(is_local_host_ip(pBriefServers->ip_addr) &&
port == g_server_port))
{
if (g_if_trunker_self)
{
@ -1320,32 +1476,10 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \
"I am the the trunk server %s:%d", __LINE__,
pBriefServers->ip_addr, port);
tracker_fetch_trunk_fid(pTrackerServer);
g_if_trunker_self = true;
if ((result=storage_trunk_init()) != 0)
if ((result=do_set_trunk_server_myself(pTrackerServer)) != 0)
{
return result;
}
if (g_trunk_create_file_advance &&
g_trunk_create_file_interval > 0)
{
ScheduleArray scheduleArray;
ScheduleEntry entries[1];
entries[0].id = TRUNK_FILE_CREATOR_TASK_ID;
entries[0].time_base = g_trunk_create_file_time_base;
entries[0].interval = g_trunk_create_file_interval;
entries[0].task_func = trunk_create_trunk_file_advance;
entries[0].func_args = NULL;
scheduleArray.count = 1;
scheduleArray.entries = entries;
sched_add_entries(&scheduleArray);
}
trunk_sync_thread_start_all();
}
}
else
@ -1357,46 +1491,13 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \
if (g_if_trunker_self)
{
int saved_trunk_sync_thread_count;
logWarning("file: "__FILE__", line: %d, " \
"I am the old trunk server, " \
"the new trunk server is %s:%d", \
__LINE__, g_trunk_server.connections[0].ip_addr, \
g_trunk_server.connections[0].port);
tracker_report_trunk_fid(pTrackerServer);
g_if_trunker_self = false;
saved_trunk_sync_thread_count = \
g_trunk_sync_thread_count;
if (saved_trunk_sync_thread_count > 0)
{
logInfo("file: "__FILE__", line: %d, "\
"waiting %d trunk sync " \
"threads exit ...", __LINE__, \
saved_trunk_sync_thread_count);
}
while (g_trunk_sync_thread_count > 0)
{
usleep(50000);
}
if (saved_trunk_sync_thread_count > 0)
{
logInfo("file: "__FILE__", line: %d, " \
"%d trunk sync threads exited",\
__LINE__, \
saved_trunk_sync_thread_count);
}
storage_trunk_destroy_ex(true);
if (g_trunk_create_file_advance && \
g_trunk_create_file_interval > 0)
{
sched_del_entry(TRUNK_FILE_CREATOR_TASK_ID);
}
do_unset_trunk_server_myself(pTrackerServer);
}
}
}
@ -1455,7 +1556,7 @@ static int tracker_check_response(ConnectionInfo *pTrackerServer, \
}
}
return tracker_merge_servers(pTrackerServer, \
return tracker_merge_servers(pTrackerServer, tracker_index,
pBriefServers, server_count);
}
@ -1922,9 +2023,10 @@ int tracker_report_join(ConnectionInfo *pTrackerServer, \
pTargetServer = &targetServer;
strcpy(targetServer.server.id, g_my_server_id_str);
ppFound = (FDFSStorageServer **)bsearch(&pTargetServer, \
g_sorted_storages, g_storage_count, \
sizeof(FDFSStorageServer *), storage_cmp_by_server_id);
ppFound = (FDFSStorageServer **)bsearch(&pTargetServer,
g_sorted_storages, g_storage_count,
sizeof(FDFSStorageServer *),
storage_cmp_by_server_id);
if (ppFound != NULL)
{
pReqBody->status = (*ppFound)->server.status;
@ -1935,14 +2037,14 @@ int tracker_report_join(ConnectionInfo *pTrackerServer, \
{
for (i=0; i<g_tracker_group.server_count; i++)
{
if (g_my_report_status[i].my_status == -1)
if (g_my_report_status[i].my_result == -1)
{
logInfo("file: "__FILE__", line: %d, "
"tracker server: #%d. %s:%d, g_my_report_status: %d",
__LINE__, i,
"tracker server: #%d. %s:%d, "
"my_report_result: %d", __LINE__, i,
g_tracker_group.servers[i].connections[0].ip_addr,
g_tracker_group.servers[i].connections[0].port,
g_my_report_status[i].my_status);
g_my_report_status[i].my_result);
break;
}
}
@ -1990,7 +2092,7 @@ int tracker_report_join(ConnectionInfo *pTrackerServer, \
pInBuff = (char *)&respBody;
result = fdfs_recv_response(pTrackerServer, \
&pInBuff, sizeof(respBody), &in_bytes);
g_my_report_status[tracker_index].my_status = result;
g_my_report_status[tracker_index].my_result = result;
if (result != 0)
{
logError("file: "__FILE__", line: %d, "
@ -2008,10 +2110,13 @@ int tracker_report_join(ConnectionInfo *pTrackerServer, \
__LINE__, pTrackerServer->ip_addr, \
pTrackerServer->port, \
(int)sizeof(respBody), in_bytes);
g_my_report_status[tracker_index].my_status = EINVAL;
g_my_report_status[tracker_index].my_result = EINVAL;
return EINVAL;
}
g_my_report_status[tracker_index].my_status = respBody.my_status;
tracker_check_my_status(tracker_index);
if (*(respBody.src_id) == '\0' && *g_sync_src_id != '\0')
{
return tracker_sync_notify(pTrackerServer, tracker_index);
@ -2022,8 +2127,8 @@ int tracker_report_join(ConnectionInfo *pTrackerServer, \
}
}
static int tracker_report_sync_timestamp(ConnectionInfo *pTrackerServer, \
bool *bServerPortChanged)
static int tracker_report_sync_timestamp(ConnectionInfo *pTrackerServer,
const int tracker_index, bool *bServerPortChanged)
{
char out_buff[sizeof(TrackerHeader) + (FDFS_STORAGE_ID_MAX_SIZE + 4) * \
FDFS_MAX_SERVERS_EACH_GROUP];
@ -2068,11 +2173,12 @@ static int tracker_report_sync_timestamp(ConnectionInfo *pTrackerServer, \
return result;
}
return tracker_check_response(pTrackerServer, bServerPortChanged);
return tracker_check_response(pTrackerServer, tracker_index,
bServerPortChanged);
}
static int tracker_report_df_stat(ConnectionInfo *pTrackerServer, \
bool *bServerPortChanged)
static int tracker_report_df_stat(ConnectionInfo *pTrackerServer,
const int tracker_index, bool *bServerPortChanged)
{
char out_buff[sizeof(TrackerHeader) + \
sizeof(TrackerStatReportReqBody) * 16];
@ -2115,7 +2221,7 @@ static int tracker_report_df_stat(ConnectionInfo *pTrackerServer, \
for (i=0; i<g_fdfs_store_paths.count; i++)
{
if (statvfs(g_fdfs_store_paths.paths[i], &sbuf) != 0)
if (statvfs(g_fdfs_store_paths.paths[i].path, &sbuf) != 0)
{
logError("file: "__FILE__", line: %d, " \
"call statfs fail, errno: %d, error info: %s.",\
@ -2128,12 +2234,12 @@ static int tracker_report_df_stat(ConnectionInfo *pTrackerServer, \
return errno != 0 ? errno : EACCES;
}
g_path_space_list[i].total_mb = ((int64_t)(sbuf.f_blocks) * \
g_fdfs_store_paths.paths[i].total_mb = ((int64_t)(sbuf.f_blocks) * \
sbuf.f_frsize) / FDFS_ONE_MB;
g_path_space_list[i].free_mb = ((int64_t)(sbuf.f_bavail) * \
g_fdfs_store_paths.paths[i].free_mb = ((int64_t)(sbuf.f_bavail) * \
sbuf.f_frsize) / FDFS_ONE_MB;
long2buff(g_path_space_list[i].total_mb, pStatBuff->sz_total_mb);
long2buff(g_path_space_list[i].free_mb, pStatBuff->sz_free_mb);
long2buff(g_fdfs_store_paths.paths[i].total_mb, pStatBuff->sz_total_mb);
long2buff(g_fdfs_store_paths.paths[i].free_mb, pStatBuff->sz_free_mb);
pStatBuff++;
}
@ -2147,12 +2253,12 @@ static int tracker_report_df_stat(ConnectionInfo *pTrackerServer, \
store_path_index = -1;
for (i=0; i<g_fdfs_store_paths.count; i++)
{
if (g_path_space_list[i].free_mb > \
if (g_fdfs_store_paths.paths[i].free_mb > \
g_avg_storage_reserved_mb \
&& g_path_space_list[i].free_mb > max_free_mb)
&& g_fdfs_store_paths.paths[i].free_mb > max_free_mb)
{
store_path_index = i;
max_free_mb = g_path_space_list[i].free_mb;
max_free_mb = g_fdfs_store_paths.paths[i].free_mb;
}
}
if (g_store_path_index != store_path_index)
@ -2178,11 +2284,13 @@ static int tracker_report_df_stat(ConnectionInfo *pTrackerServer, \
return result;
}
return tracker_check_response(pTrackerServer, bServerPortChanged);
return tracker_check_response(pTrackerServer, tracker_index,
bServerPortChanged);
}
static int tracker_heart_beat(ConnectionInfo *pTrackerServer, \
int *pstat_chg_sync_count, bool *bServerPortChanged)
static int tracker_heart_beat(ConnectionInfo *pTrackerServer,
const int tracker_index, int *pstat_chg_sync_count,
bool *bServerPortChanged)
{
char out_buff[sizeof(TrackerHeader) + sizeof(FDFSStorageStatBuff)];
TrackerHeader *pHeader;
@ -2308,7 +2416,84 @@ static int tracker_heart_beat(ConnectionInfo *pTrackerServer, \
return result;
}
return tracker_check_response(pTrackerServer, bServerPortChanged);
return tracker_check_response(pTrackerServer, tracker_index,
bServerPortChanged);
}
static int tracker_storage_change_status(ConnectionInfo *pTrackerServer,
const int tracker_index)
{
char out_buff[sizeof(TrackerHeader) + 8];
char in_buff[8];
TrackerHeader *pHeader;
char *pInBuff;
int result;
int leader_index;
int old_status;
int new_status;
int body_len;
int64_t nInPackLen;
leader_index = g_tracker_group.leader_index;
if (leader_index < 0 || tracker_index == leader_index)
{
return 0;
}
old_status = g_my_report_status[tracker_index].my_status;
new_status = g_my_report_status[leader_index].my_status;
if (new_status < 0 || new_status == old_status)
{
return 0;
}
logInfo("file: "__FILE__", line: %d, "
"tracker server: %s:%d, try to set storage "
"status from %d (%s) to %d (%s)", __LINE__,
pTrackerServer->ip_addr, pTrackerServer->port,
old_status, get_storage_status_caption(old_status),
new_status, get_storage_status_caption(new_status));
body_len = 1;
memset(out_buff, 0, sizeof(out_buff));
pHeader = (TrackerHeader *)out_buff;
long2buff(body_len, pHeader->pkg_len);
pHeader->cmd = TRACKER_PROTO_CMD_STORAGE_CHANGE_STATUS;
*(out_buff + sizeof(TrackerHeader)) = new_status;
if((result=tcpsenddata_nb(pTrackerServer->sock, out_buff,
sizeof(TrackerHeader) + body_len, g_fdfs_network_timeout)) != 0)
{
logError("file: "__FILE__", line: %d, "
"tracker server %s:%d, send data fail, "
"errno: %d, error info: %s.",
__LINE__, pTrackerServer->ip_addr,
pTrackerServer->port,
result, STRERROR(result));
return result;
}
pInBuff = in_buff;
result = fdfs_recv_response(pTrackerServer,
&pInBuff, sizeof(in_buff), &nInPackLen);
if (result != 0)
{
logError("file: "__FILE__", line: %d, "
"fdfs_recv_response fail, result: %d",
__LINE__, result);
return result;
}
if (nInPackLen != 0)
{
logError("file: "__FILE__", line: %d, "
"tracker server %s:%d, response body length: %d != 0",
__LINE__, pTrackerServer->ip_addr, pTrackerServer->port,
(int)nInPackLen);
return EINVAL;
}
return 0;
}
static int tracker_storage_changelog_req(ConnectionInfo *pTrackerServer)

2
storage/tracker_client_thread.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//tracker_client_thread.h

2
storage/trunk_mgr/trunk_client.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//trunk_client.c

2
storage/trunk_mgr/trunk_client.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//trunk_client.h

2
storage/trunk_mgr/trunk_free_block_checker.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//trunk_free_block_checker.c

2
storage/trunk_mgr/trunk_free_block_checker.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//trunk_free_block_checker.h

1275
storage/trunk_mgr/trunk_mem.c
View File

File diff suppressed because it is too large Load Diff

33
storage/trunk_mgr/trunk_mem.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//trunk_mem.h
@ -22,29 +22,47 @@
#include "trunk_shared.h"
#include "fdfs_shared_func.h"
#define STORAGE_TRUNK_COMPRESS_STAGE_NONE 0
#define STORAGE_TRUNK_COMPRESS_STAGE_COMPRESS_BEGIN 1
#define STORAGE_TRUNK_COMPRESS_STAGE_APPLY_DONE 2
#define STORAGE_TRUNK_COMPRESS_STAGE_SAVE_DONE 3
#define STORAGE_TRUNK_COMPRESS_STAGE_COMMIT_MERGING 4
#define STORAGE_TRUNK_COMPRESS_STAGE_COMMIT_MERGE_DONE 5
#define STORAGE_TRUNK_COMPRESS_STAGE_COMPRESS_SUCCESS 6
#define STORAGE_TRUNK_COMPRESS_STAGE_ROLLBACK_MERGING 7
#define STORAGE_TRUNK_COMPRESS_STAGE_ROLLBACK_MERGE_DONE 8
#define STORAGE_TRUNK_COMPRESS_STAGE_FINISHED 9
#ifdef __cplusplus
extern "C" {
#endif
extern int g_slot_min_size; //slot min size, such as 256 bytes
extern int g_slot_max_size; //slot max size
extern int g_trunk_alloc_alignment_size; //the alignment size for trunk alloc
extern int g_trunk_file_size; //the trunk file size, such as 64MB
extern int g_store_path_mode; //store which path mode, fetch from tracker
extern FDFSStorageReservedSpace g_storage_reserved_space; //fetch from tracker
extern int g_avg_storage_reserved_mb; //calc by above var: g_storage_reserved_mb
extern int g_store_path_index; //store to which path
extern int g_current_trunk_file_id; //current trunk file id
extern volatile int g_current_trunk_file_id; //current trunk file id
extern TimeInfo g_trunk_create_file_time_base;
extern TimeInfo g_trunk_compress_binlog_time_base;
extern int g_trunk_create_file_interval;
extern int g_trunk_compress_binlog_min_interval;
extern int g_trunk_compress_binlog_interval;
extern int g_trunk_binlog_max_backups;
extern TrackerServerInfo g_trunk_server; //the trunk server
extern bool g_if_use_trunk_file; //if use trunk file
extern bool g_trunk_create_file_advance;
extern bool g_trunk_init_check_occupying;
extern bool g_trunk_init_reload_from_binlog;
extern bool g_trunk_free_space_merge;
extern bool g_delete_unused_trunk_files;
extern int g_trunk_binlog_compress_stage;
extern bool g_if_trunker_self; //if am i trunk server
extern int64_t g_trunk_create_file_space_threshold;
extern int64_t g_trunk_total_free_space; //trunk total free space in bytes
extern volatile int64_t g_trunk_total_free_space; //trunk total free space in bytes
extern time_t g_trunk_last_compress_time;
typedef struct tagFDFSTrunkNode {
@ -60,9 +78,10 @@ typedef struct {
} FDFSTrunkSlot;
int storage_trunk_init();
int storage_trunk_destroy_ex(const bool bNeedSleep);
int storage_trunk_destroy_ex(const bool bNeedSleep,
const bool bSaveData);
#define storage_trunk_destroy() storage_trunk_destroy_ex(false)
#define storage_trunk_destroy() storage_trunk_destroy_ex(false, true)
int trunk_alloc_space(const int size, FDFSTrunkFullInfo *pResult);
int trunk_alloc_confirm(const FDFSTrunkFullInfo *pTrunkInfo, const int status);
@ -87,7 +106,9 @@ int trunk_file_delete(const char *trunk_filename, \
int trunk_create_trunk_file_advance(void *args);
int storage_delete_trunk_data_file();
int trunk_binlog_compress_func(void *args);
int storage_trunk_binlog_compress_check_recovery();
char *storage_trunk_get_data_filename(char *full_filename);

122
storage/trunk_mgr/trunk_shared.c
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//trunk_shared.c
@ -26,52 +26,66 @@
FDFSStorePaths g_fdfs_store_paths = {0, NULL};
struct base64_context g_fdfs_base64_context;
BufferInfo g_zero_buffer = {NULL, 0, 0};
void trunk_shared_init()
int trunk_shared_init()
{
base64_init_ex(&g_fdfs_base64_context, 0, '-', '_', '.');
g_zero_buffer.alloc_size = g_zero_buffer.length = 256 * 1024;
g_zero_buffer.buff = (char *)malloc(g_zero_buffer.alloc_size);
if (g_zero_buffer.buff == NULL)
{
logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail", __LINE__,
g_zero_buffer.alloc_size);
return ENOMEM;
}
char **storage_load_paths_from_conf_file_ex(IniContext *pItemContext, \
const char *szSectionName, const bool bUseBasePath, \
int *path_count, int *err_no)
memset(g_zero_buffer.buff, 0, g_zero_buffer.length);
return 0;
}
FDFSStorePathInfo *storage_load_paths_from_conf_file_ex(
IniContext *pItemContext, const char *szSectionName,
const bool bUseBasePath, int *path_count, int *err_no)
{
char item_name[64];
char **store_paths;
FDFSStorePathInfo *store_paths;
char *pPath;
int bytes;
int i;
*path_count = iniGetIntValue(szSectionName, "store_path_count",
pItemContext, 1);
if (*path_count <= 0)
{
logError("file: "__FILE__", line: %d, " \
"store_path_count: %d is invalid!", \
logError("file: "__FILE__", line: %d, "
"store_path_count: %d is invalid!",
__LINE__, *path_count);
*err_no = EINVAL;
return NULL;
}
store_paths = (char **)malloc(sizeof(char *) * (*path_count));
bytes = sizeof(FDFSStorePathInfo) * (*path_count);
store_paths = (FDFSStorePathInfo *)malloc(bytes);
if (store_paths == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, " \
"errno: %d, error info: %s", \
__LINE__, (int)sizeof(char *) * (*path_count), \
errno, STRERROR(errno));
logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail, "
"errno: %d, error info: %s", __LINE__,
bytes, errno, STRERROR(errno));
*err_no = errno != 0 ? errno : ENOMEM;
return NULL;
}
memset(store_paths, 0, sizeof(char *) * (*path_count));
memset(store_paths, 0, bytes);
pPath = iniGetStrValue(szSectionName, "store_path0", pItemContext);
if (pPath == NULL)
{
if (!bUseBasePath)
{
logError("file: "__FILE__", line: %d, " \
"conf file must have item " \
logError("file: "__FILE__", line: %d, "
"conf file must have item "
"\"store_path0\"!", __LINE__);
*err_no = ENOENT;
free(store_paths);
@ -80,13 +94,15 @@ char **storage_load_paths_from_conf_file_ex(IniContext *pItemContext, \
pPath = g_fdfs_base_path;
}
store_paths[0] = strdup(pPath);
if (store_paths[0] == NULL)
store_paths[0].path_len = strlen(pPath);
store_paths[0].path = strdup(pPath);
if (store_paths[0].path == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, " \
"errno: %d, error info: %s", \
__LINE__, (int)strlen(pPath), \
logError("file: "__FILE__", line: %d, "
"malloc %d bytes fail, "
"errno: %d, error info: %s",
__LINE__, (int)strlen(pPath),
errno, STRERROR(errno));
*err_no = errno != 0 ? errno : ENOMEM;
free(store_paths);
@ -97,12 +113,12 @@ char **storage_load_paths_from_conf_file_ex(IniContext *pItemContext, \
for (i=1; i<*path_count; i++)
{
sprintf(item_name, "store_path%d", i);
pPath = iniGetStrValue(szSectionName, item_name, \
pPath = iniGetStrValue(szSectionName, item_name,
pItemContext);
if (pPath == NULL)
{
logError("file: "__FILE__", line: %d, " \
"conf file must have item \"%s\"!", \
logError("file: "__FILE__", line: %d, "
"conf file must have item \"%s\"!",
__LINE__, item_name);
*err_no = ENOENT;
break;
@ -111,24 +127,25 @@ char **storage_load_paths_from_conf_file_ex(IniContext *pItemContext, \
chopPath(pPath);
if (!fileExists(pPath))
{
logError("file: "__FILE__", line: %d, " \
"\"%s\" can't be accessed, " \
"errno: %d, error info: %s", __LINE__, \
logError("file: "__FILE__", line: %d, "
"\"%s\" can't be accessed, "
"errno: %d, error info: %s", __LINE__,
pPath, errno, STRERROR(errno));
*err_no = errno != 0 ? errno : ENOENT;
break;
}
if (!isDir(pPath))
{
logError("file: "__FILE__", line: %d, " \
"\"%s\" is not a directory!", \
logError("file: "__FILE__", line: %d, "
"\"%s\" is not a directory!",
__LINE__, pPath);
*err_no = ENOTDIR;
break;
}
store_paths[i] = strdup(pPath);
if (store_paths[i] == NULL)
store_paths[i].path_len = strlen(pPath);
store_paths[i].path = strdup(pPath);
if (store_paths[i].path == NULL)
{
logError("file: "__FILE__", line: %d, " \
"malloc %d bytes fail, " \
@ -143,9 +160,9 @@ char **storage_load_paths_from_conf_file_ex(IniContext *pItemContext, \
{
for (i=0; i<*path_count; i++)
{
if (store_paths[i] != NULL)
if (store_paths[i].path != NULL)
{
free(store_paths[i]);
free(store_paths[i].path);
}
}
free(store_paths);
@ -163,7 +180,7 @@ int storage_load_paths_from_conf_file(IniContext *pItemContext)
pPath = iniGetStrValue(NULL, "base_path", pItemContext);
if (pPath == NULL)
{
logError("file: "__FILE__", line: %d, " \
logError("file: "__FILE__", line: %d, "
"conf file must have item \"base_path\"!", __LINE__);
return ENOENT;
}
@ -172,20 +189,20 @@ int storage_load_paths_from_conf_file(IniContext *pItemContext)
chopPath(g_fdfs_base_path);
if (!fileExists(g_fdfs_base_path))
{
logError("file: "__FILE__", line: %d, " \
"\"%s\" can't be accessed, error info: %s", \
logError("file: "__FILE__", line: %d, "
"\"%s\" can't be accessed, error info: %s",
__LINE__, STRERROR(errno), g_fdfs_base_path);
return errno != 0 ? errno : ENOENT;
}
if (!isDir(g_fdfs_base_path))
{
logError("file: "__FILE__", line: %d, " \
"\"%s\" is not a directory!", \
logError("file: "__FILE__", line: %d, "
"\"%s\" is not a directory!",
__LINE__, g_fdfs_base_path);
return ENOTDIR;
}
g_fdfs_store_paths.paths = storage_load_paths_from_conf_file_ex( \
g_fdfs_store_paths.paths = storage_load_paths_from_conf_file_ex(
pItemContext, NULL, true, &g_fdfs_store_paths.count, &result);
return result;
@ -259,7 +276,7 @@ int storage_split_filename(const char *logic_filename, \
SPLIT_FILENAME_BODY(logic_filename, filename_len, true_filename, \
store_path_index, true);
*ppStorePath = g_fdfs_store_paths.paths[store_path_index];
*ppStorePath = g_fdfs_store_paths.paths[store_path_index].path;
return 0;
}
@ -289,7 +306,7 @@ char *trunk_info_dump(const FDFSTrunkFullInfo *pTrunkInfo, char *buff, \
"store_path_index=%d, " \
"sub_path_high=%d, " \
"sub_path_low=%d, " \
"id=%d, offset=%d, size=%d, status=%d", \
"id=%u, offset=%d, size=%d, status=%d", \
pTrunkInfo->path.store_path_index, \
pTrunkInfo->path.sub_path_high, \
pTrunkInfo->path.sub_path_low, \
@ -325,7 +342,7 @@ char *trunk_get_full_filename_ex(const FDFSStorePaths *pStorePaths, \
char short_filename[64];
char *pStorePath;
pStorePath = pStorePaths->paths[pTrunkInfo->path.store_path_index];
pStorePath = pStorePaths->paths[pTrunkInfo->path.store_path_index].path;
TRUNK_GET_FILENAME(pTrunkInfo->file.id, short_filename);
snprintf(full_filename, buff_size, \
@ -378,7 +395,7 @@ void trunk_file_info_encode(const FDFSTrunkFileInfo *pTrunkFile, char *str)
int2buff(pTrunkFile->id, buff);
int2buff(pTrunkFile->offset, buff + sizeof(int));
int2buff(pTrunkFile->size, buff + sizeof(int) * 2);
base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(buff), \
base64_encode_ex(&g_fdfs_base64_context, buff, sizeof(buff),
str, &len, false);
}
@ -387,7 +404,7 @@ void trunk_file_info_decode(const char *str, FDFSTrunkFileInfo *pTrunkFile)
char buff[FDFS_TRUNK_FILE_INFO_LEN];
int len;
base64_decode_auto(&g_fdfs_base64_context, str, FDFS_TRUNK_FILE_INFO_LEN, \
base64_decode_auto(&g_fdfs_base64_context, str, FDFS_TRUNK_FILE_INFO_LEN,
buff, &len);
pTrunkFile->id = buff2int(buff);
@ -547,7 +564,7 @@ int trunk_file_do_lstat_func_ex(const FDFSStorePaths *pStorePaths, \
if (filename_len != FDFS_TRUNK_FILENAME_LENGTH) //not trunk file
{
snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \
pStorePaths->paths[store_path_index], true_filename);
pStorePaths->paths[store_path_index].path, true_filename);
if (stat_func == FDFS_STAT_FUNC_STAT)
{
@ -576,7 +593,7 @@ int trunk_file_do_lstat_func_ex(const FDFSStorePaths *pStorePaths, \
if (!IS_TRUNK_FILE(file_size)) //slave file
{
snprintf(full_filename, sizeof(full_filename), "%s/data/%s", \
pStorePaths->paths[store_path_index], true_filename);
pStorePaths->paths[store_path_index].path, true_filename);
if (stat_func == FDFS_STAT_FUNC_STAT)
{
@ -655,10 +672,13 @@ int trunk_file_do_lstat_func_ex(const FDFSStorePaths *pStorePaths, \
}
else
{
close(fd);
logError("file: "__FILE__", line: %d, " \
"Invalid file type: %d", __LINE__, \
/*
logError("file: "__FILE__", line: %d, "
"Invalid file type: %d", __LINE__,
pTrunkHeader->file_type);
*/
close(fd);
return ENOENT;
}

27
storage/trunk_mgr/trunk_shared.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//trunk_shared.h
@ -46,7 +46,21 @@
#define IS_TRUNK_FILE_BY_ID(trunkInfo) (trunkInfo.file.id > 0)
#define TRUNK_GET_FILENAME(file_id, filename) \
sprintf(filename, "%06d", file_id)
sprintf(filename, "%06u", file_id)
typedef struct
{
int total_mb; //total spaces
int free_mb; //free spaces
int path_len; //the length of store path
char *path; //file store path
char *mark; //path mark to avoid confusion
} FDFSStorePathInfo;
typedef struct {
int count; //store path count
FDFSStorePathInfo *paths; //file store paths
} FDFSStorePaths;
#ifdef __cplusplus
extern "C" {
@ -54,6 +68,7 @@ extern "C" {
extern FDFSStorePaths g_fdfs_store_paths; //file store paths
extern struct base64_context g_fdfs_base64_context; //base64 context
extern BufferInfo g_zero_buffer; //zero buffer for reset
typedef int (*stat_func)(const char *filename, struct stat *buf);
@ -84,11 +99,11 @@ typedef struct tagFDFSTrunkFullInfo {
FDFSTrunkFileInfo file;
} FDFSTrunkFullInfo;
char **storage_load_paths_from_conf_file_ex(IniContext *pItemContext, \
const char *szSectionName, const bool bUseBasePath, \
int *path_count, int *err_no);
FDFSStorePathInfo *storage_load_paths_from_conf_file_ex(
IniContext *pItemContext, const char *szSectionName,
const bool bUseBasePath, int *path_count, int *err_no);
int storage_load_paths_from_conf_file(IniContext *pItemContext);
void trunk_shared_init();
int trunk_shared_init();
int storage_split_filename(const char *logic_filename, \
int *filename_len, char *true_filename, char **ppStorePath);

1516
storage/trunk_mgr/trunk_sync.c
View File

File diff suppressed because it is too large Load Diff

25
storage/trunk_mgr/trunk_sync.h
View File

@ -3,7 +3,7 @@
*
* FastDFS may be copied only under the terms of the GNU General
* Public License V3, which may be found in the FastDFS source kit.
* Please visit the FastDFS Home Page http://www.csource.org/ for more detail.
* Please visit the FastDFS Home Page http://www.fastken.com/ for more detail.
**/
//trunk_sync.h
@ -28,8 +28,8 @@ extern "C" {
typedef struct
{
char storage_id[FDFS_STORAGE_ID_MAX_SIZE];
char mark_filename[MAX_PATH_SIZE];
BinLogBuffer binlog_buff;
int mark_fd;
int binlog_fd;
int64_t binlog_offset;
int64_t last_binlog_offset; //for write to mark file
@ -61,6 +61,8 @@ int trunk_sync_thread_start_all();
int trunk_sync_thread_start(const FDFSStorageBrief *pStorage);
int kill_trunk_sync_threads();
int trunk_binlog_sync_func(void *args);
int trunk_binlog_flush(const bool bNeedLock); //wrapper for trunk_binlog_fsync
void trunk_waiting_sync_thread_exit();
char *get_trunk_binlog_filename(char *full_filename);
char *trunk_mark_filename_by_reader(const void *pArg, char *full_filename);
@ -72,14 +74,31 @@ int trunk_rename_mark_file(const char *old_ip_addr, const int old_port, \
int trunk_open_readable_binlog(TrunkBinLogReader *pReader, \
get_filename_func filename_func, const void *pArg);
int trunk_reader_init(FDFSStorageBrief *pStorage, TrunkBinLogReader *pReader);
int trunk_reader_init(const FDFSStorageBrief *pStorage,
TrunkBinLogReader *pReader, const bool reset_binlog_offset);
void trunk_reader_destroy(TrunkBinLogReader *pReader);
//trunk binlog compress
int trunk_binlog_compress_delete_binlog_rollback_file(const bool silence);
int trunk_binlog_compress_delete_rollback_files(const bool silence);
int trunk_binlog_compress_delete_temp_files_after_commit();
int trunk_binlog_compress_apply();
int trunk_binlog_compress_commit();
int trunk_binlog_compress_rollback();
int trunk_sync_notify_thread_reset_offset();
int trunk_binlog_get_write_version();
int storage_delete_trunk_data_file();
char *get_trunk_binlog_tmp_filename_ex(const char *binlog_filename,
char *tmp_filename);
static inline char *get_trunk_binlog_tmp_filename(char *tmp_filename)
{
return get_trunk_binlog_tmp_filename_ex(NULL, tmp_filename);
}
#ifdef __cplusplus
}
#endif

Some files were not shown because too many files have changed in this diff Show More