pdxx
/
fastdfs
mirror of https://github.com/happyfish100/fastdfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

logger enhancement

pull/48/head
yuqing 2014-07-20 21:26:06 +08:00
parent 0015467136
commit 7ed787d837
10 changed files with 347 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
HISTORY
View File

@ -1,10 +1,18 @@
Version 5.02 2014-07-18
Version 5.02 2014-07-20
* corect README spell mistake
* bug fixed: can't deal sync truncate file exception
* remove tracker_global.c extern keyword to tracker_global.h
* change log level from ERROR to DEBUG when IOEVENT_ERROR
* php callback should use INIT_ZVAL to init zval variable
* add function short2buff and buff2short
* add get_url_content_ex to support buffer passed by caller
* logger can set rotate time format
* logger can log header line
* #include <stdbool.h> to use C99 bool
* logger can delete old rotated files
* bug fixed: connection pool should NOT increase counter when connect fail
* logger.c do NOT call fsync after write
Version 5.01 2014-02-02
* trunk binlog be compressed when trunk init

9
common/common_define.h
View File

@ -27,6 +27,7 @@ typedef DWORD (WINAPI *ThreadEntranceFunc)(LPVOID lpThreadParameter);
#include <unistd.h>
#include <signal.h>
#include <stdbool.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/types.h>
@ -89,14 +90,6 @@ extern int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int kind);
#define IP_ADDRESS_SIZE 16
#define INFINITE_FILE_SIZE (256 * 1024LL * 1024 * 1024 * 1024 * 1024LL)
#ifndef __cplusplus
#ifndef true
typedef char bool;
#define true 1
#define false 0
#endif
#endif
#ifndef byte
#define byte signed char
#endif

4
common/connection_pool.c
View File

@ -202,6 +202,10 @@ ConnectionInfo *conn_pool_get_connection(ConnectionPool *cp,
cp->connect_timeout);
if (*err_no != 0)
{
pthread_mutex_lock(&cm->lock);
cm->total_count--; //rollback
pthread_mutex_unlock(&cm->lock);
free(p);
return NULL;
}

187
common/http_func.c
View File

@ -22,21 +22,21 @@
#include "logger.h"
#include "shared_func.h"
int get_url_content(const char *url, const int connect_timeout, \
const int network_timeout, int *http_status, \
char **content, int *content_len, char *error_info)
int get_url_content_ex(const char *url, const int url_len,
const int connect_timeout, const int network_timeout,
int *http_status, char **content, int *content_len, char *error_info)
{
char domain_name[256];
char ip_addr[IP_ADDRESS_SIZE];
char out_buff[4096];
int domain_len;
int url_len;
int out_len;
int alloc_size;
int recv_bytes;
int result;
int sock;
int port;
bool bNeedAlloc;
const char *pDomain;
const char *pContent;
const char *pURI;
@ -44,10 +44,18 @@ int get_url_content(const char *url, const int connect_timeout, \
char *pSpace;
*http_status = 0;
if (*content == NULL)
{
bNeedAlloc = true;
alloc_size = 64 * 1024;
}
else
{
bNeedAlloc = false;
alloc_size = *content_len - 1;
}
*content_len = 0;
*content = NULL;
url_len = strlen(url);
if (url_len <= 7 || strncasecmp(url, "http://", 7) != 0)
{
sprintf(error_info, "file: "__FILE__", line: %d, " \
@ -137,43 +145,56 @@ int get_url_content(const char *url, const int connect_timeout, \
return result;
}
alloc_size = 64 * 1024;
*content = (char *)malloc(alloc_size + 1);
if (*content == NULL)
{
close(sock);
result = errno != 0 ? errno : ENOMEM;
if (bNeedAlloc)
{
*content = (char *)malloc(alloc_size + 1);
if (*content == NULL)
{
close(sock);
result = errno != 0 ? errno : ENOMEM;
sprintf(error_info, "file: "__FILE__", line: %d, " \
"malloc %d bytes fail, errno: %d, " \
"error info: %s", __LINE__, alloc_size + 1, \
result, STRERROR(result));
sprintf(error_info, "file: "__FILE__", line: %d, " \
"malloc %d bytes fail, errno: %d, " \
"error info: %s", __LINE__, alloc_size + 1, \
result, STRERROR(result));
return result;
}
return result;
}
}
do
{
recv_bytes = alloc_size - *content_len;
if (recv_bytes <= 0)
{
alloc_size *= 2;
*content = (char *)realloc(*content, alloc_size + 1);
if (*content == NULL)
{
close(sock);
result = errno != 0 ? errno : ENOMEM;
if (bNeedAlloc)
{
alloc_size *= 2;
*content = (char *)realloc(*content, alloc_size + 1);
if (*content == NULL)
{
*content_len = 0;
close(sock);
result = errno != 0 ? errno : ENOMEM;
sprintf(error_info, "file: "__FILE__", line: %d, " \
"realloc %d bytes fail, errno: %d, " \
"error info: %s", __LINE__, \
alloc_size + 1, \
result, STRERROR(result));
sprintf(error_info, "file: "__FILE__", line: %d, " \
"realloc %d bytes fail, errno: %d, " \
"error info: %s", __LINE__, \
alloc_size + 1, \
result, STRERROR(result));
return result;
}
return result;
}
recv_bytes = alloc_size - *content_len;
recv_bytes = alloc_size - *content_len;
}
else
{
sprintf(error_info, "file: "__FILE__", line: %d, " \
"buffer size: %d is too small", \
__LINE__, alloc_size);
return ENOSPC;
}
}
result = tcprecvdata_ex(sock, *content + *content_len, \
@ -182,62 +203,70 @@ int get_url_content(const char *url, const int connect_timeout, \
*content_len += recv_bytes;
} while (result == 0);
if (result != ENOTCONN)
{
close(sock);
free(*content);
*content = NULL;
*content_len = 0;
do
{
if (result == ENOTCONN)
{
result = 0;
}
else {
sprintf(error_info, "file: "__FILE__", line: %d, " \
"recv data from %s:%d fail, errno: %d, " \
"error info: %s", __LINE__, domain_name, \
port, result, STRERROR(result));
sprintf(error_info, "file: "__FILE__", line: %d, " \
"recv data from %s:%d fail, errno: %d, " \
"error info: %s", __LINE__, domain_name, \
port, result, STRERROR(result));
break;
}
return result;
}
*(*content + *content_len) = '\0';
pContent = strstr(*content, "\r\n\r\n");
if (pContent == NULL)
{
sprintf(error_info, "file: "__FILE__", line: %d, " \
"response data from %s:%d is invalid", \
__LINE__, domain_name, port);
*(*content + *content_len) = '\0';
pContent = strstr(*content, "\r\n\r\n");
if (pContent == NULL)
{
close(sock);
free(*content);
*content = NULL;
*content_len = 0;
result = EINVAL;
break;
}
sprintf(error_info, "file: "__FILE__", line: %d, " \
"response data from %s:%d is invalid", \
__LINE__, domain_name, port);
pContent += 4;
pSpace = strchr(*content, ' ');
if (pSpace == NULL || pSpace >= pContent)
{
sprintf(error_info, "file: "__FILE__", line: %d, " \
"response data from %s:%d is invalid", \
__LINE__, domain_name, port);
return EINVAL;
}
result = EINVAL;
break;
}
pContent += 4;
pSpace = strchr(*content, ' ');
if (pSpace == NULL || pSpace >= pContent)
{
close(sock);
free(*content);
*content = NULL;
*content_len = 0;
sprintf(error_info, "file: "__FILE__", line: %d, " \
"response data from %s:%d is invalid", \
__LINE__, domain_name, port);
return EINVAL;
}
*http_status = atoi(pSpace + 1);
*content_len -= pContent - *content;
memcpy(*content, pContent, *content_len);
*(*content + *content_len) = '\0';
*http_status = atoi(pSpace + 1);
*content_len -= pContent - *content;
memcpy(*content, pContent, *content_len);
*(*content + *content_len) = '\0';
*error_info = '\0';
} while (0);
close(sock);
if (result != 0 && bNeedAlloc)
{
free(*content);
*content = NULL;
*content_len = 0;
}
*error_info = '\0';
return 0;
return result;
}
int get_url_content(const char *url, const int connect_timeout, \
const int network_timeout, int *http_status, \
char **content, int *content_len, char *error_info)
{
*content = NULL;
return get_url_content_ex(url, strlen(url), connect_timeout, network_timeout,
http_status, content, content_len, error_info);
}
int http_parse_query(char *url, KeyValuePair *params, const int max_count)

22
common/http_func.h
View File

@ -24,8 +24,26 @@ extern "C" {
get content from url
params:
url: the url to fetch, must start as: "http://"
connect_timeout: connect timeout (seconds)
network_timeout: network timeout (seconds)
connect_timeout: connect timeout in seconds
network_timeout: network timeout in seconds
http_status: return http status code, 200 for Ok
content: return the content (HTTP body only, not including HTTP header),
*content should be freed by caller when input *content is NULL for auto malloc
content_len: input for *content buffer size when *content is NOT NULL,
output for content length (bytes)
return: 0 for success, != 0 for error
**/
int get_url_content_ex(const char *url, const int url_len,
const int connect_timeout, const int network_timeout,
int *http_status, char **content, int *content_len, char *error_info);
/**
get content from url
params:
url: the url to fetch, must start as: "http://"
connect_timeout: connect timeout in seconds
network_timeout: network timeout in seconds
http_status: return http status code, 200 for Ok
content: return the content (HTTP body only, not including HTTP header),
*content should be freed by caller

3
common/ini_file_reader.c
View File

@ -139,7 +139,7 @@ int iniLoadFromFile(const char *szFilename, IniContext *pContext)
int tail_len;
tail_len = pLast - szFilename;
if (len + tail_len >= sizeof( \
if (len + 1 + tail_len >= sizeof( \
pContext->config_path))
{
logError("file: "__FILE__", line: %d, "\
@ -149,6 +149,7 @@ int iniLoadFromFile(const char *szFilename, IniContext *pContext)
return ENOSPC;
}
*(pContext->config_path + len++) = '/';
memcpy(pContext->config_path + len, \
szFilename, tail_len);
len += tail_len;

132
common/logger.c
View File

@ -70,9 +70,8 @@ int log_init_ex(LogContext *pContext)
memset(pContext, 0, sizeof(LogContext));
pContext->log_level = LOG_INFO;
pContext->log_fd = STDERR_FILENO;
pContext->log_to_cache = false;
pContext->rotate_immediately = false;
pContext->time_precision = LOG_TIME_PRECISION_SECOND;
strcpy(pContext->rotate_time_format, "%Y%m%d_%H%M%S");
pContext->log_buff = (char *)malloc(LOG_BUFF_SIZE);
if (pContext->log_buff == NULL)
@ -112,6 +111,10 @@ static int log_open(LogContext *pContext)
pContext->log_filename, errno, STRERROR(errno));
return errno != 0 ? errno : EACCES;
}
if (pContext->current_size == 0 && pContext->print_header_callback != NULL)
{
pContext->print_header_callback(pContext);
}
return 0;
}
@ -148,6 +151,32 @@ void log_set_time_precision(LogContext *pContext, const int time_precision)
pContext->time_precision = time_precision;
}
void log_set_rotate_time_format(LogContext *pContext, const char *time_format)
{
snprintf(pContext->rotate_time_format,
sizeof(pContext->rotate_time_format),
"%s", time_format);
}
void log_set_keep_days(LogContext *pContext, const int keep_days)
{
pContext->keep_days = keep_days;
}
void log_set_header_callback(LogContext *pContext, LogHeaderCallback header_callback)
{
pContext->print_header_callback = header_callback;
if (pContext->print_header_callback != NULL)
{
pthread_mutex_lock(&(pContext->log_thread_lock));
if (pContext->current_size == 0)
{
pContext->print_header_callback(pContext);
}
pthread_mutex_unlock(&(pContext->log_thread_lock));
}
}
void log_destroy_ex(LogContext *pContext)
{
if (pContext->log_fd >= 0 && pContext->log_fd != STDERR_FILENO)
@ -189,11 +218,59 @@ int log_notify_rotate(void *args)
return 0;
}
int log_delete_old_files(void *args)
{
LogContext *pContext;
char old_filename[MAX_PATH_SIZE + 32];
int len;
struct tm tm;
time_t the_time;
if (args == NULL)
{
return EINVAL;
}
pContext = (LogContext *)args;
if (*(pContext->log_filename) == '\0' || \
*(pContext->rotate_time_format) == '\0')
{
return EINVAL;
}
if (pContext->keep_days <= 0) {
return 0;
}
the_time = get_current_time() - pContext->keep_days * 86400;
while (1) {
the_time -= 86400;
localtime_r(&the_time, &tm);
memset(old_filename, 0, sizeof(old_filename));
len = sprintf(old_filename, "%s.", pContext->log_filename);
strftime(old_filename + len, sizeof(old_filename) - len,
pContext->rotate_time_format, &tm);
if (unlink(old_filename) != 0)
{
if (errno != ENOENT)
{
fprintf(stderr, "file: "__FILE__", line: %d, " \
"unlink %s fail, errno: %d, error info: %s\n", \
__LINE__, old_filename, errno, STRERROR(errno));
}
break;
}
}
return 0;
}
static int log_rotate(LogContext *pContext)
{
struct tm tm;
time_t current_time;
char new_filename[MAX_PATH_SIZE + 32];
int len;
char old_filename[MAX_PATH_SIZE + 32];
if (*(pContext->log_filename) == '\0')
{
@ -204,15 +281,22 @@ static int log_rotate(LogContext *pContext)
current_time = get_current_time();
localtime_r(&current_time, &tm);
sprintf(new_filename, "%s.%04d%02d%02d_%02d%02d%02d", \
pContext->log_filename, \
tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, \
tm.tm_hour, tm.tm_min, tm.tm_sec);
if (rename(pContext->log_filename, new_filename) != 0)
memset(old_filename, 0, sizeof(old_filename));
len = sprintf(old_filename, "%s.", pContext->log_filename);
strftime(old_filename + len, sizeof(old_filename) - len,
pContext->rotate_time_format, &tm);
if (access(old_filename, F_OK) == 0)
{
fprintf(stderr, "file: "__FILE__", line: %d, " \
"file: %s already exist, rotate file fail\n",
__LINE__, old_filename);
}
else if (rename(pContext->log_filename, old_filename) != 0)
{
fprintf(stderr, "file: "__FILE__", line: %d, " \
"rename %s to %s fail, errno: %d, error info: %s", \
__LINE__, pContext->log_filename, new_filename, \
"rename %s to %s fail, errno: %d, error info: %s\n", \
__LINE__, pContext->log_filename, old_filename, \
errno, STRERROR(errno));
}
@ -307,6 +391,7 @@ static int log_fsync(LogContext *pContext, const bool bNeedLock)
break;
}
/*
if (pContext->log_fd != STDERR_FILENO)
{
if (fsync(pContext->log_fd) != 0)
@ -318,6 +403,7 @@ static int log_fsync(LogContext *pContext, const bool bNeedLock)
break;
}
}
*/
if (pContext->rotate_immediately)
{
@ -340,7 +426,7 @@ static int log_fsync(LogContext *pContext, const bool bNeedLock)
static void doLogEx(LogContext *pContext, struct timeval *tv, \
const char *caption, const char *text, const int text_len, \
const bool bNeedSync)
const bool bNeedSync, const bool bNeedLock)
{
struct tm tm;
int time_fragment;
@ -364,7 +450,7 @@ static void doLogEx(LogContext *pContext, struct timeval *tv, \
}
localtime_r(&tv->tv_sec, &tm);
if ((result=pthread_mutex_lock(&pContext->log_thread_lock)) != 0)
if (bNeedLock && (result=pthread_mutex_lock(&pContext->log_thread_lock)) != 0)
{
fprintf(stderr, "file: "__FILE__", line: %d, " \
"call pthread_mutex_lock fail, " \
@ -377,7 +463,10 @@ static void doLogEx(LogContext *pContext, struct timeval *tv, \
fprintf(stderr, "file: "__FILE__", line: %d, " \
"log buff size: %d < log text length: %d ", \
__LINE__, LOG_BUFF_SIZE, text_len + 64);
pthread_mutex_unlock(&(pContext->log_thread_lock));
if (bNeedLock)
{
pthread_mutex_unlock(&(pContext->log_thread_lock));
}
return;
}
@ -417,7 +506,7 @@ static void doLogEx(LogContext *pContext, struct timeval *tv, \
log_fsync(pContext, false);
}
if ((result=pthread_mutex_unlock(&(pContext->log_thread_lock))) != 0)
if (bNeedLock && (result=pthread_mutex_unlock(&(pContext->log_thread_lock))) != 0)
{
fprintf(stderr, "file: "__FILE__", line: %d, " \
"call pthread_mutex_unlock fail, " \
@ -426,8 +515,9 @@ static void doLogEx(LogContext *pContext, struct timeval *tv, \
}
}
static void doLog(LogContext *pContext, const char *caption, \
const char *text, const int text_len, const bool bNeedSync)
void log_it_ex2(LogContext *pContext, const char *caption, \
const char *text, const int text_len, \
const bool bNeedSync, const bool bNeedLock)
{
struct timeval tv;
@ -441,7 +531,7 @@ static void doLog(LogContext *pContext, const char *caption, \
gettimeofday(&tv, NULL);
}
doLogEx(pContext, &tv, caption, text, text_len, bNeedSync);
doLogEx(pContext, &tv, caption, text, text_len, bNeedSync, bNeedLock);
}
void log_it_ex1(LogContext *pContext, const int priority, \
@ -490,7 +580,7 @@ void log_it_ex1(LogContext *pContext, const int priority, \
break;
}
doLog(pContext, caption, text, text_len, bNeedSync);
log_it_ex2(pContext, caption, text, text_len, bNeedSync, true);
}
void log_it_ex(LogContext *pContext, const int priority, const char *format, ...)
@ -545,7 +635,7 @@ void log_it_ex(LogContext *pContext, const int priority, const char *format, ...
break;
}
doLog(pContext, caption, text, len, bNeedSync);
log_it_ex2(pContext, caption, text, len, bNeedSync, true);
}
@ -565,7 +655,7 @@ void log_it_ex(LogContext *pContext, const int priority, const char *format, ...
va_end(ap); \
} \
\
doLog(pContext, caption, text, len, bNeedSync); \
log_it_ex2(pContext, caption, text, len, bNeedSync, true); \
void logEmergEx(LogContext *pContext, const char *format, ...)
@ -619,7 +709,7 @@ void logAccess(LogContext *pContext, struct timeval *tvStart, \
len = vsnprintf(text, sizeof(text), format, ap);
va_end(ap);
doLogEx(pContext, tvStart, NULL, text, len, false);
doLogEx(pContext, tvStart, NULL, text, len, false, true);
}
#ifndef LOG_FORMAT_CHECK

63
common/logger.h
View File

@ -22,6 +22,11 @@ extern "C" {
#define LOG_TIME_PRECISION_MSECOND 'm' //millisecond
#define LOG_TIME_PRECISION_USSECOND 'u' //microsecond
struct log_context;
//log header line callback
typedef void (*LogHeaderCallback)(struct log_context *pContext);
typedef struct log_context
{
/* log level value please see: sys/syslog.h
@ -61,6 +66,19 @@ typedef struct log_context
/* save the log filename */
char log_filename[MAX_PATH_SIZE];
/* the time format for rotated filename,
* default: %Y%m%d_%H%M%S
* */
char rotate_time_format[32];
/* keep days for rotated log files */
int keep_days;
/*
* log the header (title line) callback
* */
LogHeaderCallback print_header_callback;
} LogContext;
extern LogContext g_log_context;
@ -122,6 +140,30 @@ void log_set_cache_ex(LogContext *pContext, const bool bLogCache);
*/
void log_set_time_precision(LogContext *pContext, const int time_precision);
/** set rotate time format, the time format same as function strftime
* parameters:
* pContext: the log context
* time_format: rotate time format
* return: none
*/
void log_set_rotate_time_format(LogContext *pContext, const char *time_format);
/** set keep days
* parameters:
* pContext: the log context
* keep_days: the keep days
* return: none
*/
void log_set_keep_days(LogContext *pContext, const int keep_days);
/** set print header callback
* parameters:
* pContext: the log context
* header_callback: the callback
* return: none
*/
void log_set_header_callback(LogContext *pContext, LogHeaderCallback header_callback);
/** destroy function
* parameters:
* pContext: the log context
@ -152,6 +194,20 @@ void log_it_ex(LogContext *pContext, const int priority, \
void log_it_ex1(LogContext *pContext, const int priority, \
const char *text, const int text_len);
/** log to file
* parameters:
* pContext: the log context
* caption: such as INFO, ERROR, NULL for no caption
* text: text string to log
* text_len: text string length (bytes)
* bNeedSync: if sync to file immediatelly
* return: none
*/
void log_it_ex2(LogContext *pContext, const char *caption, \
const char *text, const int text_len, \
const bool bNeedSync, const bool bNeedLock);
/** sync log buffer to log file
* parameters:
* args: should be (LogContext *)
@ -166,6 +222,13 @@ int log_sync_func(void *args);
*/
int log_notify_rotate(void *args);
/** delete old log files
* parameters:
* args: should be (LogContext *)
* return: error no, 0 for success, != 0 fail
*/
int log_delete_old_files(void *args);
void logEmergEx(LogContext *pContext, const char *format, ...);
void logCritEx(LogContext *pContext, const char *format, ...);
void logAlertEx(LogContext *pContext, const char *format, ...);

14
common/shared_func.c
View File

@ -1140,6 +1140,20 @@ int safeWriteToFile(const char *filename, const char *buff, \
return 0;
}
void short2buff(const short n, char *buff)
{
unsigned char *p;
p = (unsigned char *)buff;
*p++ = (n >> 8) & 0xFF;
*p++ = n & 0xFF;
}
short buff2short(const char *buff)
{
return (short)((((unsigned char)(*(buff))) << 8) | \
((unsigned char)(*(buff+1))));
}
void int2buff(const int n, char *buff)
{
unsigned char *p;

15
common/shared_func.h
View File

@ -147,6 +147,21 @@ char *hex2bin(const char *s, char *szBinBuff, int *nDestLen);
*/
void printBuffHex(const char *s, const int len);
/** 16 bits int convert to buffer (big-endian)
* parameters:
* n: 16 bits int value
* buff: the buffer, at least 2 bytes space, no tail \0
* return: none
*/
void short2buff(const short n, char *buff);
/** buffer convert to 16 bits int
* parameters:
* buff: big-endian 2 bytes buffer
* return: 16 bits int value
*/
short buff2short(const char *buff);
/** 32 bits int convert to buffer (big-endian)
* parameters:
* n: 32 bits int value