add get_url_content_ex to support buffer passed by caller

pull/1/head
yuqing 2014-06-13 14:13:06 +08:00
parent b0494666ad
commit b620f77391
4 changed files with 131 additions and 83 deletions

View File

@ -1,9 +1,12 @@
Version 1.06 2014-06-06 Version 1.06 2014-06-13
* update source code from FastDFS V5.02 * update source code from FastDFS V5.02
* add function short2buff and buff2short * add function short2buff and buff2short
* add object memory pool (fast_mblock.h and fast_mblock.c) * add object memory pool (fast_mblock.h and fast_mblock.c)
* add member thread_loop_callback in nio_thread_data * add member thread_loop_callback in nio_thread_data
* bug fixed: ini_file_reader.c can't include relative path sub config,
such as #include ../../conf/common.conf
* add get_url_content_ex to support buffer passed by caller
Version 1.05 2012-07-08 Version 1.05 2012-07-08
* update source code from FastDFS V3.09 * update source code from FastDFS V3.09

View File

@ -22,21 +22,21 @@
#include "logger.h" #include "logger.h"
#include "shared_func.h" #include "shared_func.h"
int get_url_content(const char *url, const int connect_timeout, \ int get_url_content_ex(const char *url, const int url_len,
const int network_timeout, int *http_status, \ const int connect_timeout, const int network_timeout,
char **content, int *content_len, char *error_info) int *http_status, char **content, int *content_len, char *error_info)
{ {
char domain_name[256]; char domain_name[256];
char ip_addr[IP_ADDRESS_SIZE]; char ip_addr[IP_ADDRESS_SIZE];
char out_buff[4096]; char out_buff[4096];
int domain_len; int domain_len;
int url_len;
int out_len; int out_len;
int alloc_size; int alloc_size;
int recv_bytes; int recv_bytes;
int result; int result;
int sock; int sock;
int port; int port;
bool bNeedAlloc;
const char *pDomain; const char *pDomain;
const char *pContent; const char *pContent;
const char *pURI; const char *pURI;
@ -44,10 +44,18 @@ int get_url_content(const char *url, const int connect_timeout, \
char *pSpace; char *pSpace;
*http_status = 0; *http_status = 0;
if (*content == NULL)
{
bNeedAlloc = true;
alloc_size = 64 * 1024;
}
else
{
bNeedAlloc = false;
alloc_size = *content_len;
}
*content_len = 0; *content_len = 0;
*content = NULL;
url_len = strlen(url);
if (url_len <= 7 || strncasecmp(url, "http://", 7) != 0) if (url_len <= 7 || strncasecmp(url, "http://", 7) != 0)
{ {
sprintf(error_info, "file: "__FILE__", line: %d, " \ sprintf(error_info, "file: "__FILE__", line: %d, " \
@ -137,7 +145,8 @@ int get_url_content(const char *url, const int connect_timeout, \
return result; return result;
} }
alloc_size = 64 * 1024; if (bNeedAlloc)
{
*content = (char *)malloc(alloc_size + 1); *content = (char *)malloc(alloc_size + 1);
if (*content == NULL) if (*content == NULL)
{ {
@ -151,16 +160,20 @@ int get_url_content(const char *url, const int connect_timeout, \
return result; return result;
} }
}
do do
{ {
recv_bytes = alloc_size - *content_len; recv_bytes = alloc_size - *content_len;
if (recv_bytes <= 0) if (recv_bytes <= 0)
{
if (bNeedAlloc)
{ {
alloc_size *= 2; alloc_size *= 2;
*content = (char *)realloc(*content, alloc_size + 1); *content = (char *)realloc(*content, alloc_size + 1);
if (*content == NULL) if (*content == NULL)
{ {
*content_len = 0;
close(sock); close(sock);
result = errno != 0 ? errno : ENOMEM; result = errno != 0 ? errno : ENOMEM;
@ -175,6 +188,14 @@ int get_url_content(const char *url, const int connect_timeout, \
recv_bytes = alloc_size - *content_len; recv_bytes = alloc_size - *content_len;
} }
else
{
sprintf(error_info, "file: "__FILE__", line: %d, " \
"buffer size: %d is too small", \
__LINE__, alloc_size);
return ENOSPC;
}
}
result = tcprecvdata_ex(sock, *content + *content_len, \ result = tcprecvdata_ex(sock, *content + *content_len, \
recv_bytes, network_timeout, &recv_bytes); recv_bytes, network_timeout, &recv_bytes);
@ -182,62 +203,67 @@ int get_url_content(const char *url, const int connect_timeout, \
*content_len += recv_bytes; *content_len += recv_bytes;
} while (result == 0); } while (result == 0);
do
{
if (result != ENOTCONN) if (result != ENOTCONN)
{ {
close(sock);
free(*content);
*content = NULL;
*content_len = 0;
sprintf(error_info, "file: "__FILE__", line: %d, " \ sprintf(error_info, "file: "__FILE__", line: %d, " \
"recv data from %s:%d fail, errno: %d, " \ "recv data from %s:%d fail, errno: %d, " \
"error info: %s", __LINE__, domain_name, \ "error info: %s", __LINE__, domain_name, \
port, result, STRERROR(result)); port, result, STRERROR(result));
return result; break;
} }
*(*content + *content_len) = '\0'; *(*content + *content_len) = '\0';
pContent = strstr(*content, "\r\n\r\n"); pContent = strstr(*content, "\r\n\r\n");
if (pContent == NULL) if (pContent == NULL)
{ {
close(sock);
free(*content);
*content = NULL;
*content_len = 0;
sprintf(error_info, "file: "__FILE__", line: %d, " \ sprintf(error_info, "file: "__FILE__", line: %d, " \
"response data from %s:%d is invalid", \ "response data from %s:%d is invalid", \
__LINE__, domain_name, port); __LINE__, domain_name, port);
return EINVAL; result = EINVAL;
break;
} }
pContent += 4; pContent += 4;
pSpace = strchr(*content, ' '); pSpace = strchr(*content, ' ');
if (pSpace == NULL || pSpace >= pContent) if (pSpace == NULL || pSpace >= pContent)
{ {
close(sock);
free(*content);
*content = NULL;
*content_len = 0;
sprintf(error_info, "file: "__FILE__", line: %d, " \ sprintf(error_info, "file: "__FILE__", line: %d, " \
"response data from %s:%d is invalid", \ "response data from %s:%d is invalid", \
__LINE__, domain_name, port); __LINE__, domain_name, port);
return EINVAL; result = EINVAL;
break;
} }
*http_status = atoi(pSpace + 1); *http_status = atoi(pSpace + 1);
*content_len -= pContent - *content; *content_len -= pContent - *content;
memcpy(*content, pContent, *content_len); memcpy(*content, pContent, *content_len);
*(*content + *content_len) = '\0'; *(*content + *content_len) = '\0';
*error_info = '\0';
} while (0);
close(sock); close(sock);
if (result != 0 && bNeedAlloc)
{
free(*content);
*content = NULL;
*content_len = 0;
}
*error_info = '\0'; return result;
return 0; }
int get_url_content(const char *url, const int connect_timeout, \
const int network_timeout, int *http_status, \
char **content, int *content_len, char *error_info)
{
*content = NULL;
return get_url_content_ex(url, strlen(url), connect_timeout, network_timeout,
http_status, content, content_len, error_info);
} }
int http_parse_query(char *url, KeyValuePair *params, const int max_count) int http_parse_query(char *url, KeyValuePair *params, const int max_count)

View File

@ -24,8 +24,26 @@ extern "C" {
get content from url get content from url
params: params:
url: the url to fetch, must start as: "http://" url: the url to fetch, must start as: "http://"
connect_timeout: connect timeout (seconds) connect_timeout: connect timeout in seconds
network_timeout: network timeout (seconds) network_timeout: network timeout in seconds
http_status: return http status code, 200 for Ok
content: return the content (HTTP body only, not including HTTP header),
*content should be freed by caller when input *content is NULL for auto malloc
content_len: input for *content buffer size when *content is NOT NULL,
output for content length (bytes)
return: 0 for success, != 0 for error
**/
int get_url_content_ex(const char *url, const int url_len,
const int connect_timeout, const int network_timeout,
int *http_status, char **content, int *content_len, char *error_info);
/**
get content from url
params:
url: the url to fetch, must start as: "http://"
connect_timeout: connect timeout in seconds
network_timeout: network timeout in seconds
http_status: return http status code, 200 for Ok http_status: return http status code, 200 for Ok
content: return the content (HTTP body only, not including HTTP header), content: return the content (HTTP body only, not including HTTP header),
*content should be freed by caller *content should be freed by caller

View File

@ -139,7 +139,7 @@ int iniLoadFromFile(const char *szFilename, IniContext *pContext)
int tail_len; int tail_len;
tail_len = pLast - szFilename; tail_len = pLast - szFilename;
if (len + tail_len >= sizeof( \ if (len + 1 + tail_len >= sizeof( \
pContext->config_path)) pContext->config_path))
{ {
logError("file: "__FILE__", line: %d, "\ logError("file: "__FILE__", line: %d, "\
@ -149,6 +149,7 @@ int iniLoadFromFile(const char *szFilename, IniContext *pContext)
return ENOSPC; return ENOSPC;
} }
*(pContext->config_path + len++) = '/';
memcpy(pContext->config_path + len, \ memcpy(pContext->config_path + len, \
szFilename, tail_len); szFilename, tail_len);
len += tail_len; len += tail_len;