diff --git a/app/wizard/tmpl/http/main_proc.cpp b/app/wizard/tmpl/http/main_proc.cpp index d98028d58..0fbce4b5a 100644 --- a/app/wizard/tmpl/http/main_proc.cpp +++ b/app/wizard/tmpl/http/main_proc.cpp @@ -22,7 +22,7 @@ int main(int argc, char* argv[]) acl::log::stdout_open(true); // 监听的地址列表,格式:ip:port1,ip:port2,... - const char* addrs = "127.0.0.1:8888"; + const char* addrs = ":8888"; printf("listen on: %s\r\n", addrs); // 测试时设置该值 > 0 则指定服务器处理客户端连接过程的 diff --git a/app/wizard/tmpl/http/main_threads.cpp b/app/wizard/tmpl/http/main_threads.cpp index 18ee29f74..968255ea9 100644 --- a/app/wizard/tmpl/http/main_threads.cpp +++ b/app/wizard/tmpl/http/main_threads.cpp @@ -22,7 +22,7 @@ int main(int argc, char* argv[]) acl::log::stdout_open(true); // 监听的地址列表,格式:ip:port1,ip:port2,... - const char* addrs = "127.0.0.1:8888"; + const char* addrs = ":8888"; printf("listen on: %s\r\n", addrs); // 测试时设置该值 > 0 则指定服务器处理客户端连接过程的 diff --git a/app/wizard/tmpl/master/main_aio.cpp b/app/wizard/tmpl/master/main_aio.cpp index 059a083fb..99c871ab6 100644 --- a/app/wizard/tmpl/master/main_aio.cpp +++ b/app/wizard/tmpl/master/main_aio.cpp @@ -22,7 +22,7 @@ int main(int argc, char* argv[]) acl::log::stdout_open(true); // 监听的地址列表,格式:ip:port1,ip:port2,... - const char* addrs = "127.0.0.1:8888"; + const char* addrs = ":8888"; printf("listen on: %s\r\n", addrs); // 单独运行方式 diff --git a/app/wizard/tmpl/master/main_proc.cpp b/app/wizard/tmpl/master/main_proc.cpp index d98028d58..0fbce4b5a 100644 --- a/app/wizard/tmpl/master/main_proc.cpp +++ b/app/wizard/tmpl/master/main_proc.cpp @@ -22,7 +22,7 @@ int main(int argc, char* argv[]) acl::log::stdout_open(true); // 监听的地址列表,格式:ip:port1,ip:port2,... - const char* addrs = "127.0.0.1:8888"; + const char* addrs = ":8888"; printf("listen on: %s\r\n", addrs); // 测试时设置该值 > 0 则指定服务器处理客户端连接过程的 diff --git a/app/wizard/tmpl/master/main_threads.cpp b/app/wizard/tmpl/master/main_threads.cpp index 18ee29f74..968255ea9 100644 --- a/app/wizard/tmpl/master/main_threads.cpp +++ b/app/wizard/tmpl/master/main_threads.cpp @@ -22,7 +22,7 @@ int main(int argc, char* argv[]) acl::log::stdout_open(true); // 监听的地址列表,格式:ip:port1,ip:port2,... - const char* addrs = "127.0.0.1:8888"; + const char* addrs = ":8888"; printf("listen on: %s\r\n", addrs); // 测试时设置该值 > 0 则指定服务器处理客户端连接过程的 diff --git a/app/wizard/tmpl/master/main_udp.cpp b/app/wizard/tmpl/master/main_udp.cpp index 58fdb60e8..c72b02055 100644 --- a/app/wizard/tmpl/master/main_udp.cpp +++ b/app/wizard/tmpl/master/main_udp.cpp @@ -22,7 +22,7 @@ int main(int argc, char* argv[]) acl::log::stdout_open(true); // 监听的地址列表,格式:ip:port1,ip:port2,... - const char* addrs = "127.0.0.1:8888"; + const char* addrs = ":8888"; printf("bind on: %s\r\n", addrs); // 测试时设置该值 > 0 则指定服务器处理客户端连接过程的 diff --git a/lib_acl_cpp/changes.txt b/lib_acl_cpp/changes.txt index 25d7536b8..d4fb4ccdf 100644 --- a/lib_acl_cpp/changes.txt +++ b/lib_acl_cpp/changes.txt @@ -1,6 +1,17 @@ 修改历史列表: ------------------------------------------------------------------------ +346) 2015.8.18 +346.1) feature: http_client/http_request 类在读取压缩数据体时,会自动处理 +临时处理结果为 0 的情况,以降低使用复杂度 + +345) 2015.8.17 +345.1) http_client gzip 数据压缩传输测试通过 + +344) 2015.8.16 +344.1) feature: http_client 类支持响应 gzip 压缩类型的数据 +344.2) feature: HttpServletResponse 类中的输出流对象采用了统一了 http_client 类型 + 343) 2015.8.15 343.1) bugfix: redis_string::incoper 内部在区分 INCR 与 INCRBY, DECR 与 DECRBY 的方法有误 diff --git a/lib_acl_cpp/include/acl_cpp/http/HttpServlet.hpp b/lib_acl_cpp/include/acl_cpp/http/HttpServlet.hpp index d77df9efe..bb1ee9332 100644 --- a/lib_acl_cpp/include/acl_cpp/http/HttpServlet.hpp +++ b/lib_acl_cpp/include/acl_cpp/http/HttpServlet.hpp @@ -20,8 +20,8 @@ class ACL_CPP_API HttpServlet virtual ~HttpServlet(void) = 0; /** - * 设置本地字符集,如果设置了本地字符集,则在接收 HTTP 请求数据时,会自动将请求的 - * 字符集转为本地字符集;该函数必须在 doRun 之前调用才有效 + * 设置本地字符集,如果设置了本地字符集,则在接收 HTTP 请求数据时,会 + * 自动将请求的字符集转为本地字符集;该函数必须在 doRun 之前调用才有效 * @param charset {const char*} 本地字符集,如果该指针为空, * 则清除本地字符集 * @return {HttpServlet&} @@ -29,26 +29,27 @@ class ACL_CPP_API HttpServlet HttpServlet& setLocalCharset(const char* charset); /** - * 设置 HTTP 会话过程的 IO 读写超时时间;该函数必须在 doRun 之前调用才有效 + * 设置 HTTP 会话过程 IO 读写超时时间;该函数必须在 doRun 前调用才有效 * @param rw_timeout {int} 读写超时时间(秒) * @return {HttpServlet&} */ HttpServlet& setRwTimeout(int rw_timeout); /** - * 针对 POST 方法,该方法设置是否需要解析数据体数据,默认为解析,该函数必须在 doRun - * 之前调用才有效;当数据体为数据流或 MIME 格式,即使调用本方法设置了解析数据,也不 - * 会对数据体进行解析 + * 针对 POST 方法,该方法设置是否需要解析数据体数据,默认为解析,该函 + * 数必须在 doRun 之前调用才有效;当数据体为数据流或 MIME 格式,即使 + * 调用本方法设置了解析数据,也不会对数据体进行解析 * @param on {bool} 是否需要解析 * @return {HttpServlet&} */ HttpServlet& setParseBody(bool on); /** - * 针对 POST 方法,该方法设置解析数据体的最大长度,如果数据体,该函数必须在 doRun - * 之前调用才有效 - * @param length {int} 最大长度限制,如果请求的数据体长度过大,则直接返回 - * false,如果该值 <= 0 则内部不限制数据体长度,调用该函数前内部缺省值为 0 + * 针对 POST 方法,该方法设置解析数据体的最大长度,如果数据体,该函数 + * 必须在 doRun 之前调用才有效 + * @param length {int} 最大长度限制,如果请求的数据体长度过大,则直接 + * 返回 false,如果该值 <= 0 则内部不限制数据体长度,调用该函数前 + * 内部缺省值为 0 * @return {HttpServlet&} */ HttpServlet& setParseBodyLimit(int length); diff --git a/lib_acl_cpp/include/acl_cpp/http/HttpServletRequest.hpp b/lib_acl_cpp/include/acl_cpp/http/HttpServletRequest.hpp index 946b2e081..b906eaad6 100644 --- a/lib_acl_cpp/include/acl_cpp/http/HttpServletRequest.hpp +++ b/lib_acl_cpp/include/acl_cpp/http/HttpServletRequest.hpp @@ -1,6 +1,7 @@ #pragma once #include "acl_cpp/acl_cpp_define.hpp" #include +#include "acl_cpp/stdlib/string.hpp" #include "acl_cpp/http/http_header.hpp" #include "acl_cpp/http/http_ctype.hpp" #include "acl_cpp/http/http_type.hpp" @@ -135,10 +136,11 @@ class ACL_CPP_API HttpServletRequest #endif /** - * 如果客户端的请求是分段数据,则该函数将获得请求头中的长度起始地址及结束地址 + * 如果客户端的请求是分段数据,则该函数将获得请求头中的长度起始地址 + * 及结束地址 * @param range_from {long long int&} 偏移起始位置 * @param range_to {long long int&} 偏移结束位置 - * @return {bool} 若出错或非分段请求则返回 false,若是分段请求则返回 true + * @return {bool} 若出错或非分段请求则返回false,若是分段请求则返回true * 注:range_from/range_to 下标从 0 开始 */ #if defined(_WIN32) || defined(_WIN64) @@ -278,6 +280,12 @@ class ACL_CPP_API HttpServletRequest */ int getKeepAlive(void) const; + /** + * 获得 HTTP 客户端支持的数据压缩算法集合 + * @param out {std::vector&} 存储结果集 + */ + void getAcceptEncoding(std::vector& out) const; + /* * 当 HTTP 请求为 POST 方法,通过本函数设置读 HTTP 数据体的 * IO 超时时间值(秒) diff --git a/lib_acl_cpp/include/acl_cpp/http/HttpServletResponse.hpp b/lib_acl_cpp/include/acl_cpp/http/HttpServletResponse.hpp index 393a8344e..083231b84 100644 --- a/lib_acl_cpp/include/acl_cpp/http/HttpServletResponse.hpp +++ b/lib_acl_cpp/include/acl_cpp/http/HttpServletResponse.hpp @@ -7,8 +7,10 @@ class string; class ostream; class socket_stream; class http_header; +class http_client; class HttpCookie; class HttpServlet; +class HttpServletRequest; /** * 与 HTTP 客户端响应相关的类,该类不应被继承,用户也不需要 @@ -46,6 +48,7 @@ class ACL_CPP_API HttpServletResponse /** * 设置与 HTTP 客户端保持联系长连接 * @param on {bool} + * @return {HttpServletResponse&} */ HttpServletResponse& setKeepAlive(bool on); @@ -53,13 +56,22 @@ class ACL_CPP_API HttpServletResponse * 设置 HTTP 响应数据体的 Content-Type 字段值,可字段值可以为: * text/html 或 text/html; charset=utf8 格式 * @param value {const char*} 字段值 + * @return {HttpServletResponse&} */ HttpServletResponse& setContentType(const char* value); + /** + * 设置 HTTP 响应数据体采用 gzip 压缩格式 + * @param gzip {bool} 是否采用 gzip 压缩格式 + * @return {HttpServletResponse&} + */ + HttpServletResponse& setContentEncoding(bool gzip); + /** * 设置 HTTP 响应数据体中字符集,当已经在 setContentType 设置 * 了字符集,则就不必再调用本函数设置字符集 * @param charset {const char*} 响应体数据的字符集 + * @return {HttpServletResponse&} */ HttpServletResponse& setCharacterEncoding(const char* charset); @@ -88,8 +100,8 @@ class ACL_CPP_API HttpServletResponse * 对于分区下载,调用本函数设置数据下载的偏移位置(下标从 0 开始) * @param from {http_off_t} 数据区间起始偏移位置(下标从 0 开始计算) * @param to {http_off_t} 数据区间结束位置(该值需小于总数据长度) - * @param total {http_off_t} 总数据长度,当数据源为一个静态文件时该值应 - * 等于该文件的总长度大小 + * @param total {http_off_t} 总数据长度,当数据源为一个静态文件时该值 + * 应等于该文件的总长度大小 * @return {HttpServletResponse&} */ #if defined(_WIN32) || defined(_WIN64) @@ -178,8 +190,9 @@ class ACL_CPP_API HttpServletResponse /** * 带格式方式向 HTTP 客户端发送响应数据,内部自动调用 - * HttpServletResponse::write(const void*, size_t) 过程, - * 在使用 chunked 方式传输数据时,应该应该最后再调用 write(NULL, 0) 表示数据结束 + * HttpServletResponse::write(const void*, size_t) 过程,在使用 + * chunked 方式传输数据时,应该应该最后再调用 write(NULL, 0) + * 表示数据结束 * @param fmt {const char*} 变参格式字符串 * @return {int} 成功则返回值 > 0,否则返回 -1 */ @@ -187,8 +200,8 @@ class ACL_CPP_API HttpServletResponse /** * 带格式方式向 HTTP 客户端发送响应数据,内部自动调用 - * HttpServletResponse::write(const string&) 过程, - * 在使用 chunked 方式传输数据时,应该应该最后再调用 write(NULL, 0) 表示数据结束 + * HttpServletResponse::write(const string&) 过程,在使用 chunked + * 方式传输数据时,应该应该最后再调用 write(NULL, 0) 表示数据结束 * @param fmt {const char*} 变参格式字符串 * @param ap {va_list} 变参列表 * @return {int} 成功则返回值 > 0,否则返回 -1 @@ -214,8 +227,16 @@ class ACL_CPP_API HttpServletResponse */ ostream& getOutputStream(void) const; + /** + * 设置 http 请求对象,该函数目前只应被 HttpServlet 类内部调用 + * @param request {HttpServletRequest*} + */ + void setHttpServletRequest(HttpServletRequest* request); + private: socket_stream& stream_; // 客户端连接流 + HttpServletRequest* request_; // http 请求对象 + http_client* client_; // http 响应流对象 http_header* header_; // http 响应头 char charset_[32]; // 字符集 char content_type_[32]; // content-type 类型 diff --git a/lib_acl_cpp/include/acl_cpp/http/http_client.hpp b/lib_acl_cpp/include/acl_cpp/http/http_client.hpp index 572ac8efd..8b1d3a789 100644 --- a/lib_acl_cpp/include/acl_cpp/http/http_client.hpp +++ b/lib_acl_cpp/include/acl_cpp/http/http_client.hpp @@ -69,9 +69,9 @@ class ACL_CPP_API http_client /** * 写 HTTP 请求头数据至输出流中 * @param header {http_header&} - * @return {int} 真实写入的数据量, 返回 -1 表示出错 + * @return {bool} 写头部数据是否成功 */ - int write_head(const http_header& header); + bool write_head(const http_header& header); /** * 发送 HTTP 数据体,可以循环调用此函数,当在第一次调用 write 函数写入 @@ -282,11 +282,15 @@ class ACL_CPP_API http_client * @param clean {bool} 在接收数据前是否自动清空 buf 缓冲区 * @param real_size {int*} 若该指针非空,则记录真正读到的数据长度, * 通过该指针返回的数据值永远 >= 0 - * @return {int} 返回值含义如下: + * @return {int} 返回值含义如下:(应用需要通过 body_finish 函数和 + * disconnected 函数来判断数据体是否读完或连接是否关闭) * > 0: 表示已经读到的数据,并且数据还未读完 - * == 0: 如果返回值为此值,则可以调用 disconnected()函数来判断连接 - * 是否已经关闭;调用 body_finish 函数来判断是否已经读完 HTTP 响应 - * 体数据,如果已经读完且连接未关闭,则还可以继续保持长连接 + * == 0: 有两种原因会返回 0,当数据读完时返回 0,可调用 body_finish + * 函数判断是否已经读完 HTTP 响应数据;当读到压缩数据的尾部时, + * 因压缩数据的8字节尾部数据是控制字段,所以不做为数据体返回, + * 此时也会返回 0; + * 还可以通过 disconnected() 函数判断连接是否已经被关闭 + * 如果数据读完且连接半未关闭,则可以继续保持长连接 * < 0: 表示连接关闭 * 注:read_body 的两个函数不能混用; * 当为解压缩数据时,则返回的值为解压缩后的数据长度 @@ -312,7 +316,7 @@ class ACL_CPP_API http_client * 关闭,当返回 true 时表示读到了一行数据,此时可以通过判断 * body_finish() 返回值来判断是否已经读完了数据体 * @param out {string&} 存储数据体的缓冲区,在该函数内部不会自动清理该 - * 缓冲区,用户可在调用该函数前自行清理该缓冲区中的数据(可调用:out.clear()) + * 缓冲区,用户可在调用该函数前自行清理该缓冲区(可调用:out.clear()) * @param nonl {bool} 读取一行数据时是否自动去掉尾部的 "\r\n" 或 "\n" * @param size {size_t*} 当读到完整的一行数据时存放该行数据的长度, * 当读到一个空行且 nonl 为 true 时,则该值为 0 @@ -386,9 +390,12 @@ class ACL_CPP_API http_client bool is_request_; // 是否是客户请求端 int gzip_header_left_; // gzip 头剩余的长度 int last_ret_; // 数据读完后记录最后的返回值 + bool head_sent_; // 头部数据是否已经发送完毕 bool body_finish_; // 是否已经读完 HTTP 响应体数据 bool disconnected_; // 网络连接是否已经关闭 bool chunked_transfer_; // 是否为 chunked 传输模式 + unsigned gzip_crc32_; // gzip 压缩数据时的检验值 + unsigned gzip_total_in_; // gzip 压缩前的总数据长度 string* buf_; // 内部缓冲区,用在按行读等操作中 bool read_request_head(void); @@ -399,6 +406,11 @@ class ACL_CPP_API http_client int read_response_body(string& out, bool clean, int* real_size); HTTP_HDR* get_http_hdr() const; + bool write_chunk(ostream& out, const void* data, size_t len); + bool write_chunk_trailer(ostream& out); + + bool write_gzip(ostream& out, const void* data, size_t len); + bool write_gzip_trailer(ostream& out); }; } // namespace acl diff --git a/lib_acl_cpp/include/acl_cpp/http/http_header.hpp b/lib_acl_cpp/include/acl_cpp/http/http_header.hpp index 75019a4bd..576a38161 100644 --- a/lib_acl_cpp/include/acl_cpp/http/http_header.hpp +++ b/lib_acl_cpp/include/acl_cpp/http/http_header.hpp @@ -183,9 +183,9 @@ class ACL_CPP_API http_header */ bool is_request(void) const; - ////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////// // HTTP 请求方法函数 - ////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////// /** * 创建 HTTP 请求头数据 @@ -313,9 +313,9 @@ class ACL_CPP_API http_header */ virtual void redicrect_reset(void) {} - ////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////// // HTTP 响应方法函数 - ////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////// /** * 创建 HTTP 响应头数据 @@ -354,9 +354,34 @@ class ACL_CPP_API http_header */ http_header& set_cgi_mode(bool on); + /** + * 是否设置了 CGI 模式 + * @return {bool} + */ + bool is_cgi_mode() const + { + return cgi_mode_; + } + + /** + * 设置传输的数据是否采用 gzip 方式进行压缩 + * @param on {bool} + * @return {http_header&} + */ + http_header& set_transfer_gzip(bool on); + + /** + * 获得当前的数据传输是否设置了采用 gzip 压缩方式 + * @return {bool} + */ + bool is_transfer_gzip() const + { + return transfer_gzip_; + } + private: //char* domain_; // HTTP 服务器域名 - //unsigned short port_; // HTTP 服务器端口 + //unsigned short port_; // HTTP 服务器端口 char* url_; // HTTP 请求的 URL std::list params_; // 请求参数集合 std::list cookies_; // cookies 集合 @@ -382,6 +407,7 @@ class ACL_CPP_API http_header long long int content_length_; // HTTP 数据体长度 #endif bool chunked_transfer_; // 是否为 chunked 传输模式 + bool transfer_gzip_; // 数据是否采用 gzip 压缩 void init(void); // 初始化 void clear(void); void build_common(string& buf) const; // 构建通用头 diff --git a/lib_acl_cpp/include/acl_cpp/http/http_request.hpp b/lib_acl_cpp/include/acl_cpp/http/http_request.hpp index 2751cfb23..94f2671a6 100644 --- a/lib_acl_cpp/include/acl_cpp/http/http_request.hpp +++ b/lib_acl_cpp/include/acl_cpp/http/http_request.hpp @@ -202,20 +202,22 @@ class ACL_CPP_API http_request : public connect_client * 与真实读到的数据不同,真实读到的数据长度应该通过参数 real_size 来 * 获得); < 0 表示数据流关闭,此时若 real_size 非空,则 real_size 存 * 储的值应该为 0 + * 当返回 0 时,可调用 body_finish 函数判断是否读完了所有数据体 */ int read_body(string& out, bool clean = false, int* real_size = NULL); /** - * 当调用 request 成功后调用本函数来从 HTTP 服务端读一行数据,可以循环调用 + * 调用 request 成功后调用本函数来从 HTTP 服务端读一行数据,可循环调用 * 本函数,直到返回 false 或 body_finish() 返回 true 为止; - * 本函数内部自动对压缩数据进行解压,如果在调用本函数之前调用 set_charset 设置了 + * 内部自动对压缩数据解压,如果在调用本函数之前调用 set_charset 设置了 * 本地字符集,则还同时对数据进行字符集转码操作 * @param out {string&} 存储结果数据 * @param nonl {bool} 读到的一行数据是否自动去掉尾部的 "\r\n" 或 "\n" * @param size {size_t*} 该指针非空时存放读到的数据长度 - * @return {bool} 是否读到了一行数据:当返回 true 时表示读到了一行数据,可以 - * 通过 body_finish() 是否为 true 来判断是否读数据体已经结束,当读到一个空行 - * 且 nonl = true 时,则 *size = 0;当返回 false 时表示未读完整行且读完毕, + * @return {bool} 是否读到一行数据:返回 true 时表示读到了一行数据, + * 可以通过 body_finish() 是否为 true 来判断是否读数据体已经结束, + * 当读到一个空行 且 nonl = true 时,则 *size = 0;当返回 false 时 + * 表示未读完整行且读完毕, * *size 中存放着读到的数据长度 */ bool body_gets(string& out, bool nonl = true, size_t* size = NULL); @@ -295,7 +297,7 @@ class ACL_CPP_API http_request : public connect_client protected: /** - * 基类 connect_client 的纯虚函数,显式地调用本函数用来打开与服务端的连接 + * 基类 connect_client 纯虚函数,显式调用本函数用来打开与服务端的连接 * @return {bool} 连接是否成功 */ virtual bool open(); diff --git a/lib_acl_cpp/include/acl_cpp/http/http_response.hpp b/lib_acl_cpp/include/acl_cpp/http/http_response.hpp index a9211f8ef..9837dfe96 100644 --- a/lib_acl_cpp/include/acl_cpp/http/http_response.hpp +++ b/lib_acl_cpp/include/acl_cpp/http/http_response.hpp @@ -125,7 +125,7 @@ class ACL_CPP_API http_response * 关闭 HTTP 连接流 */ void close(void); -protected: + private: bool debug_; bool header_ok_; diff --git a/lib_acl_cpp/include/acl_cpp/stdlib/zlib_stream.hpp b/lib_acl_cpp/include/acl_cpp/stdlib/zlib_stream.hpp index 995332ea1..7c6c6854c 100644 --- a/lib_acl_cpp/include/acl_cpp/stdlib/zlib_stream.hpp +++ b/lib_acl_cpp/include/acl_cpp/stdlib/zlib_stream.hpp @@ -27,6 +27,38 @@ typedef enum zlib_level9 = zlib_best_compress } zlib_level_t; +/** + * 压缩过程中的压缩窗口参数类型,值越大则压缩效果越好且占用内存越多, + * 针对 HTTP 压缩传输,需要设置这些值的负值:-zlib_wbits_t + */ +enum +{ + zlib_wbits_8 = 8, + zlib_wbits_9 = 9, + zlib_wbits_10 = 10, + zlib_wbits_11 = 11, + zlib_wbits_12 = 12, + zlib_wbits_13 = 13, + zlib_wbits_14 = 14, + zlib_wbits_15 = 15, +}; + +/** + * 压缩过程中的内存分配策略,值越大使用内存越多 + */ +typedef enum +{ + zlib_mlevel_1 = 1, + zlib_mlevel_2 = 2, + zlib_mlevel_3 = 3, + zlib_mlevel_4 = 4, + zlib_mlevel_5 = 5, + zlib_mlevel_6 = 6, + zlib_mlevel_7 = 7, + zlib_mlevel_8 = 8, + zlib_mlevel_9 = 9, +} zlib_mlevel_t; + /** * 压缩或解压过程中的缓存模式,即在压缩或解压过程中是否立刻刷新 * 到缓冲区为了获得比较高的压缩比,应该选择 zlib_flush_off 方式 @@ -85,10 +117,17 @@ class ACL_CPP_API zlib_stream : public pipe_stream * 过程失败,则应该调用 zip_reset * @param level {zlib_level_t} 压缩级别,级别越高,则压缩比 * 越高,但压缩速度越低 + * @param wbits {zlib_wbits_t} 压缩过程中的滑动窗口级别,值越大,则 + * 压缩效率越高且使用内存越多,针对 HTTP 数据压缩传输,应该采用该 + * 值的负值,如:-zlib_wbits_15 + * @param mlevel {zlib_mlevel_t} 压缩过程中的内存分配策略,值越大, + * 则压缩效率越高且内存使用越多 * @return {bool} 压缩初始化过程是否成功,失败的原因一般 * 应该是输入的参数非法 */ - bool zip_begin(zlib_level_t level = zlib_default); + bool zip_begin(zlib_level_t level = zlib_default, + int wbits = zlib_wbits_15, + zlib_mlevel_t mlevel = zlib_mlevel_9); /** * 循环调用此函数对源数据进行压缩 @@ -176,6 +215,15 @@ class ACL_CPP_API zlib_stream : public pipe_stream */ bool unzip_reset(); + /** + * 获得当前的 zstream 对象 + * @return {z_stream*} + */ + z_stream* get_zstream() const + { + return zstream_; + } + /////////////////////////////////////////////////////////////// bool pipe_zip_begin(zlib_level_t level = zlib_default, @@ -188,7 +236,7 @@ class ACL_CPP_API zlib_stream : public pipe_stream string* out, size_t max = 0); virtual int pop_end(string* out, size_t max = 0); virtual void clear(); -protected: + private: z_stream* zstream_; bool finished_; diff --git a/lib_acl_cpp/samples/http/Makefile b/lib_acl_cpp/samples/http/Makefile index ef68d9c56..07b9dd619 100644 --- a/lib_acl_cpp/samples/http/Makefile +++ b/lib_acl_cpp/samples/http/Makefile @@ -4,8 +4,10 @@ all: @(cd http_request; make) @(cd http_response; make) @(cd http_servlet; make) + @(cd cgi_end; make) clean: @(cd http_request; make clean) @(cd http_response; make clean) @(cd http_servlet; make clean) + @(cd cgi_end; make clean) diff --git a/lib_acl_cpp/samples/http/cgi_env/Makefile b/lib_acl_cpp/samples/http/cgi_env/Makefile new file mode 100644 index 000000000..1339b01c6 --- /dev/null +++ b/lib_acl_cpp/samples/http/cgi_env/Makefile @@ -0,0 +1,4 @@ +base_path = ../../.. +PROG = cgi_env +EXTLIBS = -lz +include ../../Makefile.in diff --git a/lib_acl_cpp/samples/http/cgi_env/http_servlet.cpp b/lib_acl_cpp/samples/http/cgi_env/http_servlet.cpp new file mode 100644 index 000000000..fad5823da --- /dev/null +++ b/lib_acl_cpp/samples/http/cgi_env/http_servlet.cpp @@ -0,0 +1,114 @@ +// http_servlet.cpp : 定义控制台应用程序的入口点。 +// + +#include "stdafx.h" + +using namespace acl; + +////////////////////////////////////////////////////////////////////////// + +class http_servlet : public HttpServlet +{ +public: + http_servlet(void) + { + + } + + ~http_servlet(void) + { + + } + + virtual bool doUnknown(HttpServletRequest&, HttpServletResponse& res) + { + res.setStatus(400); + res.setContentType("text/xml; charset=gb2312"); + // 发送 http 响应头 + if (res.sendHeader() == false) + return false; + // 发送 http 响应体 + string buf("\r\n"); + (void) res.getOutputStream().write(buf); + return false; + } + + virtual bool doGet(HttpServletRequest& req, HttpServletResponse& res) + { + return doPost(req, res); + } + + virtual bool doPost(HttpServletRequest& req, HttpServletResponse& res) + { + extern char **environ; + if (environ == NULL) + return doError(req, res); + + acl::string buf; + for (size_t i = 0; environ[i] != NULL; i++) + buf << environ[i] << "\r\n"; + + // 创建 HTTP 响应头 + res.setStatus(200); + res.setContentType("text/plain"); + res.setCharacterEncoding("gb2312"); + res.setContentEncoding(true); + //res.setContentLength(buf.size()); + res.setChunkedTransferEncoding(true); + + return res.write(buf) && res.write(NULL, 0); + } +}; + +////////////////////////////////////////////////////////////////////////// + +static void do_run(socket_stream* stream) +{ + memcache_session session("127.0.0.1:11211"); + http_servlet servlet; + servlet.setLocalCharset("gb2312"); + servlet.doRun(session, stream); +} + +// 服务器方式运行时的服务类 +class master_service : public master_proc +{ +public: + master_service() {} + ~master_service() {} +protected: + virtual void on_accept(socket_stream* stream) + { + do_run(stream); + } +}; + +// WEB 服务模式 +static void do_alone(void) +{ + master_service service; + printf("listen: 0.0.0.0:8888 ...\r\n"); + service.run_alone("0.0.0.0:8888", NULL, 0); // 单独运行方式 +} + +// WEB CGI 模式 +static void do_cgi(void) +{ + do_run(NULL); +} + +int main(int argc, char* argv[]) +{ +#ifdef WIN32 + acl::acl_cpp_init(); +#endif + + // 开始运行 + if (argc >= 2 && strcmp(argv[1], "alone") == 0) + do_alone(); + else + do_cgi(); + + return 0; +} + diff --git a/lib_acl_cpp/samples/http/cgi_env/stdafx.cpp b/lib_acl_cpp/samples/http/cgi_env/stdafx.cpp new file mode 100644 index 000000000..348bb1940 --- /dev/null +++ b/lib_acl_cpp/samples/http/cgi_env/stdafx.cpp @@ -0,0 +1,8 @@ +// stdafx.cpp : 只包括标准包含文件的源文件 +// cgi_upload.pch 将成为预编译头 +// stdafx.obj 将包含预编译类型信息 + +#include "stdafx.h" + +// TODO: 在 STDAFX.H 中 +//引用任何所需的附加头文件,而不是在此文件中引用 diff --git a/lib_acl_cpp/samples/http/cgi_env/stdafx.h b/lib_acl_cpp/samples/http/cgi_env/stdafx.h new file mode 100644 index 000000000..13419b886 --- /dev/null +++ b/lib_acl_cpp/samples/http/cgi_env/stdafx.h @@ -0,0 +1,12 @@ +// stdafx.h : 标准系统包含文件的包含文件, +// 或是常用但不常更改的项目特定的包含文件 +// + +#pragma once + + +//#include +//#include + +// TODO: 在此处引用程序要求的附加头文件 +#include "acl_cpp/lib_acl.hpp" diff --git a/lib_acl_cpp/samples/http/http_request/main.cpp b/lib_acl_cpp/samples/http/http_request/main.cpp index 3336b1894..9782fcdcb 100644 --- a/lib_acl_cpp/samples/http/http_request/main.cpp +++ b/lib_acl_cpp/samples/http/http_request/main.cpp @@ -3,15 +3,20 @@ static void usage(const char* procname) { - printf("usage: %s -h [help] -s server_addr[127.0.0.1:8194] -n max_loop\r\n", procname); + printf("usage: %s -h [help] \r\n" + "\t-s server_addr[127.0.0.1:8194] \r\n" + "\t-n max_loop \r\n" + "\t-z[accept_gzip]\r\n" + "\t-B[send_body data]\r\n", procname); } int main(int argc, char* argv[]) { int ch, max = 10; + bool accept_gzip = false, send_body = false; acl::string addr("127.0.0.1:8194"); - while ((ch = getopt(argc, argv, "hs:n:")) > 0) + while ((ch = getopt(argc, argv, "hs:n:zB")) > 0) { switch (ch) { @@ -24,11 +29,19 @@ int main(int argc, char* argv[]) case 'n': max = atoi(optarg); break; + case 'z': + accept_gzip = true; + break; + case 'B': + send_body = true; + break; default: break; } } + acl::log::stdout_open(true); + acl::string buf(1024); for (size_t i = 0; i < 1024; i++) buf << 'X'; @@ -39,35 +52,57 @@ int main(int argc, char* argv[]) for (int i = 0; i < max; i++) { acl::http_header& header = req.request_header(); + //header.set_method(acl::HTTP_METHOD_POST); header.set_url("/"); header.set_keep_alive(true); - header.set_content_length(buf.length()); + header.accept_gzip(accept_gzip ? true : false); + //header.set_content_length(buf.length()); + + bool rc; + + if (send_body) + rc = req.request(buf.c_str(), buf.length()); + else + rc = req.request(NULL, 0); - if (req.request(buf.c_str(), buf.length()) == false) + // 鍙墍浠ュ皢 build_request 鏀惧湪 req.request 鍚庨潰锛屾槸鍥犱负 + // req.request 鍐呴儴鍙兘浼氫慨鏀硅姹傚ご涓殑瀛楁 + acl::string hdr; + header.build_request(hdr); + printf("request header:\r\n%s\r\n", hdr.c_str()); + + if (rc == false) { printf("send request error\n"); break; } - if (i < 10) - printf("send request ok\r\n"); + printf("send request ok\r\n"); tmp.clear(); - int size = 0; + int size = 0, real_size = 0, n; while (true) { - int ret = req.read_body(tmp, false); + int ret = req.read_body(tmp, false, &n); if (ret < 0) { printf("read_body error\n"); return 1; } else if (ret == 0) + { + printf("-------------read over---------\r\n"); break; + } + size += ret; + real_size += n; } - if (i < 10) - printf(">>size: %d\n", size); + + printf("read body size: %d, real_size: %d, %s\n", + size, real_size, tmp.c_str()); + + printf("===============================================\r\n"); } return 0; diff --git a/lib_acl_cpp/samples/http/http_request/valgrind.sh b/lib_acl_cpp/samples/http/http_request/valgrind.sh new file mode 100644 index 000000000..ff563ec34 --- /dev/null +++ b/lib_acl_cpp/samples/http/http_request/valgrind.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +valgrind --tool=memcheck --leak-check=yes -v ./http_request -s 127.0.0.1:8888 -n 200 -z diff --git a/lib_acl_cpp/samples/http/http_response/main.cpp b/lib_acl_cpp/samples/http/http_response/main.cpp index 0808c8f46..528fd971a 100644 --- a/lib_acl_cpp/samples/http/http_response/main.cpp +++ b/lib_acl_cpp/samples/http/http_response/main.cpp @@ -1,47 +1,8 @@ #include "stdafx.h" #include -static void usage(const char* procname) -{ - printf("usage: %s -h [help] -s listen_addr[127.0.0.1:8194]\r\n", procname); -} - -int main(int argc, char* argv[]) +static void handle_request(acl::socket_stream* conn) { - int ch; - acl::string addr("127.0.0.1:8194"); - - while ((ch = getopt(argc, argv, "hs:")) > 0) - { - switch (ch) - { - case 'h': - usage(argv[0]); - return 0; - case 's': - addr = optarg; - break; - default: - break; - } - } - - acl::server_socket server; - if (server.open(addr) == false) - { - printf("listen error, addr: %s\r\n", addr.c_str()); - return 1; - } - printf("listen %s ok\r\n", addr.c_str()); - - acl::socket_stream* conn = server.accept(); - if (conn == NULL) - { - printf("accept error\r\n"); - return 1; - } - printf("accept one: %s\r\n", conn->get_peer()); - acl::string buf(1024); for (size_t i = 0; i < 1024; i++) buf << 'X'; @@ -59,8 +20,7 @@ int main(int argc, char* argv[]) break; } - if (n < 10) - printf("read header ok\r\n"); + printf("read header ok\r\n"); tmp.clear(); if (res.get_body(tmp) == false) @@ -69,24 +29,85 @@ int main(int argc, char* argv[]) break; } - if (n < 10) - printf(">>size: %d\r\n", (int) tmp.length()); + printf("request body's size: %d\r\n", (int) tmp.length()); acl::http_header& header = res.response_header(); header.set_status(200); header.set_keep_alive(true); - header.set_content_length(buf.length()); + header.set_transfer_gzip(true); + header.set_chunked(true); + // header.set_content_length(buf.length()); + + acl::string hdr; + header.build_response(hdr); + printf("response header:\r\n%s\r\n", hdr.c_str()); if (res.response(buf.c_str(), buf.length()) == false) { printf("response error\r\n"); break; } + if (res.response(NULL, 0) == false) + { + printf("response trailer error\r\n"); + break; + } + + printf("response body ok\r\n"); + printf("===============================================\r\n"); n++; } conn->close(); + printf(">>>> close client <<<<\r\n"); +} + +static void usage(const char* procname) +{ + printf("usage: %s -h [help] -s listen_addr[127.0.0.1:8194]\r\n", procname); +} + +int main(int argc, char* argv[]) +{ + int ch; + acl::string addr("0.0.0.0:8194"); + + while ((ch = getopt(argc, argv, "hs:")) > 0) + { + switch (ch) + { + case 'h': + usage(argv[0]); + return 0; + case 's': + addr = optarg; + break; + default: + break; + } + } + + acl::server_socket server; + if (server.open(addr) == false) + { + printf("listen error, addr: %s\r\n", addr.c_str()); + return 1; + } + printf("listen %s ok\r\n", addr.c_str()); + + while (true) + { + acl::socket_stream* conn = server.accept(); + if (conn == NULL) + { + printf("accept error\r\n"); + return 1; + } + printf("accept one: %s\r\n", conn->get_peer()); + + handle_request(conn); + } return 0; } diff --git a/lib_acl_cpp/samples/http/http_servlet/http_servlet.cpp b/lib_acl_cpp/samples/http/http_servlet/http_servlet.cpp index 85f8e1df4..c85094dc7 100644 --- a/lib_acl_cpp/samples/http/http_servlet/http_servlet.cpp +++ b/lib_acl_cpp/samples/http/http_servlet/http_servlet.cpp @@ -35,6 +35,7 @@ bool http_servlet::doGet(acl::HttpServletRequest& req, bool http_servlet::doPost(acl::HttpServletRequest& req, acl::HttpServletResponse& res) { +#if 0 const char* session_name = req.getSession().getAttribute("session_name"); if (*session_name == 0) { @@ -47,6 +48,9 @@ bool http_servlet::doPost(acl::HttpServletRequest& req, if (*session_user == 0) req.getSession().setAttribute("session_user", "user"); session_user = req.getSession().getAttribute("session_user"); +#else + const char* session_name = "name", *session_user = "user"; +#endif // 取得浏览器 cookie const char* cookie_name = req.getCookieValue("cookie_name"); @@ -89,6 +93,7 @@ bool http_servlet::doPost(acl::HttpServletRequest& req, #else res.setContentType("text/xml; charset=utf-8") // 设置响应字符集 .setKeepAlive(keep_alive) // 设置是否保持长连接 + .setContentEncoding(true) // 设置是否压缩数据 .setChunkedTransferEncoding(true); // 采用 chunk 传输方式 #endif @@ -96,5 +101,5 @@ bool http_servlet::doPost(acl::HttpServletRequest& req, // 发送 http 响应体,因为设置了 chunk 传输模式,所以需要多调用一次 // res.write 且两个参数均为 0 以表示 chunk 传输数据结束 - return res.write(buf) && res.write(NULL, 0) && keep_alive; + return res.write(buf) && res.write(NULL, 0); } diff --git a/lib_acl_cpp/src/http/HttpServlet.cpp b/lib_acl_cpp/src/http/HttpServlet.cpp index 61794f910..76462f00f 100644 --- a/lib_acl_cpp/src/http/HttpServlet.cpp +++ b/lib_acl_cpp/src/http/HttpServlet.cpp @@ -89,19 +89,22 @@ bool HttpServlet::doRun(session& session, socket_stream* stream /* = NULL */) HttpServletRequest req(res, session, *in, local_charset_, parse_body_enable_, parse_body_limit_); + // 设置 HttpServletRequest 对象 + res.setHttpServletRequest(&req); + if (rw_timeout_ >= 0) req.setRwTimeout(rw_timeout_); res.setCgiMode(cgi_mode); - bool ret; - http_method_t method = req.getMethod(); // 根据请求的值自动设定是否需要保持长连接 if (!cgi_mode) res.setKeepAlive(req.isKeepAlive()); + bool ret; + switch (method) { case HTTP_METHOD_GET: diff --git a/lib_acl_cpp/src/http/HttpServletRequest.cpp b/lib_acl_cpp/src/http/HttpServletRequest.cpp index edfec218f..3add29bab 100644 --- a/lib_acl_cpp/src/http/HttpServletRequest.cpp +++ b/lib_acl_cpp/src/http/HttpServletRequest.cpp @@ -755,6 +755,31 @@ int HttpServletRequest::getKeepAlive(void) const return atoi(ptr); } +void HttpServletRequest::getAcceptEncoding(std::vector& out) const +{ + out.clear(); + const char* ptr; + + if (cgi_mode_) + ptr = acl_getenv("HTTP_ACCEPT_ENCODING"); + else if (client_) + ptr = client_->header_value("Accept-Encoding"); + else + return; + + if (ptr == NULL || *ptr == 0) + return; + + ACL_ARGV* tokens = acl_argv_split(ptr, ",; \t"); + ACL_ITER iter; + acl_foreach(iter, tokens) + { + const char* token = (const char*) iter.data; + out.push_back(token); + } + acl_argv_free(tokens); +} + void HttpServletRequest::setRwTimeout(int rw_timeout) { rw_timeout_ = rw_timeout; diff --git a/lib_acl_cpp/src/http/HttpServletResponse.cpp b/lib_acl_cpp/src/http/HttpServletResponse.cpp index eeb9b66fb..ce59ce521 100644 --- a/lib_acl_cpp/src/http/HttpServletResponse.cpp +++ b/lib_acl_cpp/src/http/HttpServletResponse.cpp @@ -4,7 +4,9 @@ #include "acl_cpp/stream/ostream.hpp" #include "acl_cpp/stream/socket_stream.hpp" #include "acl_cpp/http/http_header.hpp" +#include "acl_cpp/http/http_client.hpp" #include "acl_cpp/http/HttpServlet.hpp" +#include "acl_cpp/http/HttpServletRequest.hpp" #include "acl_cpp/http/HttpServletResponse.hpp" namespace acl @@ -12,7 +14,9 @@ namespace acl HttpServletResponse::HttpServletResponse(socket_stream& stream) : stream_(stream) +, request_(NULL) { + client_ = NEW http_client(&stream_, stream_.get_rw_timeout()); header_ = NEW http_header(); header_->set_request_mode(false); charset_[0] = 0; @@ -22,6 +26,7 @@ HttpServletResponse::HttpServletResponse(socket_stream& stream) HttpServletResponse::~HttpServletResponse(void) { + delete client_; delete header_; } @@ -64,6 +69,12 @@ HttpServletResponse& HttpServletResponse::setContentType(const char* value) return *this; } +HttpServletResponse& HttpServletResponse::setContentEncoding(bool gzip) +{ + header_->set_transfer_gzip(gzip); + return *this; +} + HttpServletResponse& HttpServletResponse::setDateHeader( const char* name, time_t value) { @@ -136,70 +147,46 @@ bool HttpServletResponse::sendHeader(void) head_sent_ = true; acl_assert(header_->is_request() == false); - string buf; + + char buf[256]; if (charset_[0] != 0) - buf.format("%s; charset=%s", content_type_, charset_); + safe_snprintf(buf, sizeof(buf), "%s; charset=%s", + content_type_, charset_); else - buf.format("%s", content_type_); - header_->set_content_type(buf.c_str()); + safe_snprintf(buf, sizeof(buf), "%s", content_type_); + + header_->set_content_type(buf); + + // 铏界劧鏈嶅姟绔湪鍝嶅簲澶翠腑璁剧疆浜 gzip 鍘嬬缉鏂瑰紡锛屼絾濡傛灉璇锋眰绔笉鎺ユ敹 + // gzip 鍘嬬缉鏁版嵁锛屽垯闇瑕佷粠鍝嶅簲澶翠腑绂佹 + if (header_->is_transfer_gzip() && request_) + { + bool accept_gzip = false; + std::vector tokens; + request_->getAcceptEncoding(tokens); + std::vector::const_iterator it; + + for (it = tokens.begin(); it != tokens.end(); ++it) + { + if ((*it).compare("gzip", false) == 0) + { + accept_gzip = true; + break; + } + } + + if (!accept_gzip) + header_->set_transfer_gzip(false); + } - buf.clear(); - header_->build_response(buf); - return getOutputStream().write(buf) == -1 ? false : true; + return client_->write_head(*header_); } bool HttpServletResponse::write(const void* data, size_t len) { if (!head_sent_ && sendHeader() == false) return false; - - if (header_->chunked_transfer() == false) - { - if (data == NULL || len == 0) - return true; - return stream_.write(data, len) == -1 ? false : true; - } - - if (data == NULL || len == 0) - return stream_.format("0\r\n\r\n") == -1 ? false : true; - -#if 1 - struct iovec iov[3]; - - char hdr[32]; - safe_snprintf(hdr, sizeof(hdr), "%x\r\n", (int) len); - -#ifdef MINGW - iov[0].iov_base = hdr; -#else - iov[0].iov_base = (void*) hdr; -#endif - iov[0].iov_len = strlen(hdr); - -#ifdef MINGW - iov[1].iov_base = (char*) data; -#else - iov[1].iov_base = (void*) data; -#endif - iov[1].iov_len = (int) len; - -#ifdef MINGW - iov[2].iov_base = (char*) "\r\n"; -#else - iov[2].iov_base = (void*) "\r\n"; -#endif - iov[2].iov_len = 2; - - return stream_.writev(iov, 3) == -1 ? false : true; -#else - if (stream_.format("%x\r\n", (int) len) == -1) - return false; - if (stream_.write(data, len) == -1) - return false; - if (stream_.write("\r\n", 2) == -1) - return false; - return true; -#endif + return client_->write_body(data, len); } bool HttpServletResponse::write(const string& buf) @@ -236,4 +223,9 @@ ostream& HttpServletResponse::getOutputStream(void) const return stream_; } +void HttpServletResponse::setHttpServletRequest(HttpServletRequest* request) +{ + request_ = request; +} + } // namespace acl diff --git a/lib_acl_cpp/src/http/http_client.cpp b/lib_acl_cpp/src/http/http_client.cpp index 9452e2992..466cf0fc1 100644 --- a/lib_acl_cpp/src/http/http_client.cpp +++ b/lib_acl_cpp/src/http/http_client.cpp @@ -1,5 +1,7 @@ #include "acl_stdafx.hpp" +#include "zlib.h" #include "acl_cpp/stdlib/log.hpp" +#include "acl_cpp/stdlib/snprintf.hpp" #include "acl_cpp/stdlib/zlib_stream.hpp" #include "acl_cpp/stream/ostream.hpp" #include "acl_cpp/stream/socket_stream.hpp" @@ -20,9 +22,12 @@ http_client::http_client(void) , unzip_(true) , zstream_(NULL) , is_request_(true) +, head_sent_(false) , body_finish_(false) , disconnected_(true) , chunked_transfer_(false) +, gzip_crc32_(0) +, gzip_total_in_(0) , buf_(NULL) { } @@ -39,9 +44,12 @@ http_client::http_client(socket_stream* client, int rw_timeout /* = 120 */, , unzip_(unzip) , zstream_(NULL) , is_request_(is_request) +, head_sent_(false) , body_finish_(false) , disconnected_(false) , chunked_transfer_(false) +, gzip_crc32_(0) +, gzip_total_in_(0) , buf_(NULL) { } @@ -98,8 +106,11 @@ void http_client::reset(void) } last_ret_ = -1; + head_sent_ = false; body_finish_ = false; chunked_transfer_ = false; + gzip_crc32_ = 0; + gzip_total_in_ = 0; } bool http_client::open(const char* addr, int conn_timeout /* = 60 */, @@ -130,60 +141,320 @@ bool http_client::open(const char* addr, int conn_timeout /* = 60 */, return true; } -int http_client::write_head(const http_header& header) +////////////////////////////////////////////////////////////////////////////// + +bool http_client::write_chunk(ostream& out, const void* data, size_t len) +{ +#ifndef HAS_IOV +# define HAS_IOV +#endif + +#ifdef HAS_IOV + struct iovec iov[3]; + + char hdr[32]; + safe_snprintf(hdr, sizeof(hdr), "%x\r\n", (int) len); + +#ifdef MINGW + iov[0].iov_base = hdr; +#else + iov[0].iov_base = (void*) hdr; +#endif + iov[0].iov_len = strlen(hdr); + +#ifdef MINGW + iov[1].iov_base = (char*) data; +#else + iov[1].iov_base = (void*) data; +#endif + iov[1].iov_len = (int) len; + +#ifdef MINGW + iov[2].iov_base = (char*) "\r\n"; +#else + iov[2].iov_base = (void*) "\r\n"; +#endif + iov[2].iov_len = 2; + + if (out.writev(iov, 3) == -1) + { + disconnected_ = true; + return false; + } + else + return true; +#else + if (out.format("%x\r\n", (int) len) == -1 + || out.write(data, len) == -1 + || out.write("\r\n", 2) == -1) + { + disconnected_ = true; + return false; + } + else + return true; +#endif +} + +bool http_client::write_chunk_trailer(ostream& out) +{ + static const char trailer[] = "0\r\n\r\n"; + + if (out.write(trailer, sizeof(trailer) - 1) == -1) + { + disconnected_ = true; + return false; + } + else + return true; +} + +bool http_client::write_gzip(ostream& out, const void* data, size_t len) +{ + acl_assert(zstream_); + + if (buf_ == NULL) + buf_ = NEW string(4096); + else + buf_->clear(); + + // 边压缩边输出数据 + if (data && len > 0) + { + // 增加非压缩数据总长度 + gzip_total_in_ += len; + + // 计算 crc32 数据校验和 + gzip_crc32_ = crc32(gzip_crc32_, (const Bytef*) data, + (unsigned) len); + + if (!zstream_->zip_update((const char*) data, len, buf_)) + { + logger_error("zip_update error!"); + return false; + } + + // 如果为空,则直接返回,等待下次的写操作 + if (buf_->empty()) + return true; + + data = buf_->c_str(); + len = buf_->size(); + } + + // 写入 zstream 流对象中最后可能缓存的数据,同时结束压缩过程 + else + { + // 检查数据长度有效 + unsigned total_in = zstream_->get_zstream()->total_in; + if (total_in != gzip_total_in_) + logger_warn("total_in: %d != gzip_total_in_: %d", + total_in, gzip_total_in_); + + if (!zstream_->zip_finish(buf_)) + { + logger_error("zip_finish error!"); + return false; + } + + if (buf_->empty()) + return true; + + data = buf_->c_str(); + len = buf_->size(); + } + + // 块传输方式输出压缩数据 + if (chunked_transfer_) + return write_chunk(out, data, len); + + // 普通流式方式输出压缩数据 + if (out.write(data, len) < 0) + { + disconnected_ = true; + return false; + } + + return true; +} + +// 输出 gzip 尾部结束数据 + +bool http_client::write_gzip_trailer(ostream& out) +{ +#ifdef HAVE_BIG_ENDIAN + struct gztrailer + { + unsigned char crc32_[4]; + unsigned char zlen_[4]; + }; + struct gztrailer trailer; + + trailer.crc32_[0] = (u_char) (gzip_crc32_ & 0xff); + trailer.crc32_[1] = (u_char) ((gzip_crc32_ >> 8) & 0xff); + trailer.crc32_[2] = (u_char) ((gzip_crc32_ >> 16) & 0xff); + trailer.crc32_[3] = (u_char) ((gzip_crc32_ >> 24) & 0xff); + + trailer.zlen_[0] = (u_char) (gzip_total_in_ & 0xff); + trailer.zlen_[1] = (u_char) ((gzip_total_in_ >> 8) & 0xff); + trailer.zlen_[2] = (u_char) ((gzip_total_in_ >> 16) & 0xff); + trailer.zlen_[3] = (u_char) ((gzip_total_in_ >> 24) & 0xff); +#else + struct gztrailer + { + unsigned int crc32_; + unsigned int zlen_; + }; + + struct gztrailer trailer; + trailer.crc32_ = gzip_crc32_; + trailer.zlen_ = gzip_total_in_; + +#endif // HAVE_BIG_ENDIAN + + // 块传输方式输出 gzip 尾 + if (chunked_transfer_) + return write_chunk(out, &trailer, sizeof(trailer)) + && write_chunk_trailer(out); + + // 普通方式输出 gzip 尾 + if (out.write(&trailer, sizeof(trailer)) < 0) + { + disconnected_ = true; + return false; + } + else + return true; +} + +bool http_client::write_head(const http_header& header) { + if (head_sent_) + return true; + head_sent_ = true; + + // 先保留是否为块传输的状态 chunked_transfer_ = header.chunked_transfer(); + + // 如果设置了 gzip 传输方式,则需要先初始化 zlib 流对象 + if (header.is_transfer_gzip()) + { + if (zstream_ != NULL) + delete zstream_; + + zstream_ = NEW zlib_stream; + if (zstream_->zip_begin(zlib_default, -zlib_wbits_15, + zlib_mlevel_9) == false) + { + logger_error("zip_begin error!"); + delete zstream_; + zstream_ = NULL; + + // 如果初始化 zip 失败,则强制转换成非 zip 模式 + const_cast + (&header)->set_transfer_gzip(false); + } + + // 初始化 crc32 校验和 + gzip_crc32_ = crc32(0, Z_NULL, 0); + // 初始化非压缩数据总长度 + gzip_total_in_ = 0; + } + + // 创建 HTTP 请求/响应头 string buf; if (header.is_request()) header.build_request(buf); else header.build_response(buf); - int ret = get_ostream().write(buf.c_str(), buf.length()); - if (ret < 0) + + ostream& out = get_ostream(); + + // 先写 HTTP 头 + if (out.write(buf.c_str(), buf.length()) < 0) + { disconnected_ = true; - return ret; + return false; + } + else if (zstream_ == NULL) + return true; + + // 如果是采用 gzip 数据压缩方式,则需要先输出 gzip 头 + + /** + * RFC 1952 Section 2.3 defines the gzip header: + * +---+---+---+---+---+---+---+---+---+---+ + * |ID1|ID2|CM |FLG| MTIME |XFL|OS | + * +---+---+---+---+---+---+---+---+---+---+ + * Unix OS_CODE: 3 + */ + static const unsigned char gzheader[10] = + { 0x1f, 0x8b, Z_DEFLATED, 0, 0, 0, 0, 0, 0, 3 }; + + if (chunked_transfer_) + { + // 块传输方式写数据 + if (write_chunk(out, gzheader, sizeof(gzheader)) == false) + return false; + else + return true; + } + + // 普通流式写数据 + else if (out.write(gzheader, sizeof(gzheader)) < 0) + { + disconnected_ = true; + return false; + } + else + return true; } bool http_client::write_body(const void* data, size_t len) { - ostream& fp = get_ostream(); + ostream& out = get_ostream(); - if (chunked_transfer_ == false) + // 如果是 gzip 传输,则边压缩边写数据 + if (zstream_ != NULL) { - if (data == NULL || len == 0) - return true; - if (fp.write(data, len) == -1) - { - disconnected_ = true; + // 输出压缩数据体 + if (write_gzip(out, data, len) == false) return false; - } + + // 如果数据输出完毕,则还需输出 gzip 尾部字段 + if (data == NULL || len == 0) + return write_gzip_trailer(out); else return true; } - // 如果设置了 chunked 传输方式,则按块传输方式写数据 + // 非压缩方式传输数据 + // 如果参数为 NULL,则说明数据写完毕 if (data == NULL || len == 0) { - if (fp.format("0\r\n\r\n") == -1) - { - disconnected_ = true; - return false; - } + if (chunked_transfer_) + return write_chunk_trailer(out); else return true; } - if (fp.format("%x\r\n", (int) len) == -1 - || fp.write(data, len) == -1 - || fp.write("\r\n", 2) == -1) + // 块方式写入数据体 + else if (chunked_transfer_) + return write_chunk(out, data, len); + + // 普通流式写入数据体 + else if (out.write(data, len) == -1) { disconnected_ = true; return false; } - return true; + else + return true; } +////////////////////////////////////////////////////////////////////////////// + ostream& http_client::get_ostream(void) const { acl_assert(stream_); @@ -273,6 +544,8 @@ bool http_client::read_response_head(void) delete zstream_; zstream_ = NULL; } + + // gzip 响应数据体前会有 10 字节的头部字段 gzip_header_left_ = 10; } else @@ -567,14 +840,16 @@ int http_client::read_request_body(char* buf, size_t size) // 缓冲区太大了没有任何意义 if (size >= 1024000) size = 1024000; + http_off_t ret = http_req_body_get_sync(req_, vstream, buf, (int) size); - if (ret <= 0) + if (ret < 0) { + disconnected_ = true; body_finish_ = true; - if (ret < 0) - disconnected_ = true; } + else if (ret == 0) + body_finish_ = true; return ((int) ret); } @@ -598,13 +873,13 @@ int http_client::read_body(string& out, bool clean /* = true */, return read_request_body(out, clean, real_size); } -int http_client::read_response_body(string& out, bool clean, - int* real_size) +int http_client::read_response_body(string& out, bool clean, int* real_size) { if (real_size) *real_size = 0; + if (body_finish_) - return (last_ret_); + return last_ret_; if (stream_ == NULL) { @@ -612,8 +887,9 @@ int http_client::read_response_body(string& out, bool clean, disconnected_ = true; return -1; } - ACL_VSTREAM* vstream = stream_->get_vstream(); - if (vstream == NULL) + + ACL_VSTREAM* vs = stream_->get_vstream(); + if (vs == NULL) { logger_error("connect stream null"); disconnected_ = true; @@ -626,6 +902,7 @@ int http_client::read_response_body(string& out, bool clean, disconnected_ = true; return -1; } + if (res_ == NULL) res_ = http_res_new(hdr_res_); @@ -635,9 +912,9 @@ int http_client::read_response_body(string& out, bool clean, int saved_count = (int) out.length(); char buf[8192]; -SKIP_GZIP_HEAD_AGAIN: // 对于有 GZIP 头数据,可能需要重复读 +READ_AGAIN: // 对于有 GZIP 头数据,可能需要重复读 - int ret = (int) http_res_body_get_sync(res_, vstream, buf, sizeof(buf)); + int ret = (int) http_res_body_get_sync(res_, vs, buf, sizeof(buf)); if (zstream_ == NULL) { @@ -680,10 +957,11 @@ int http_client::read_response_body(string& out, bool clean, if (gzip_header_left_ >= ret) { gzip_header_left_ -= ret; - goto SKIP_GZIP_HEAD_AGAIN; + goto READ_AGAIN; } int n; + if (gzip_header_left_ > 0) { n = gzip_header_left_; @@ -691,12 +969,22 @@ int http_client::read_response_body(string& out, bool clean, } else n = 0; + if (zstream_->unzip_update(buf + n, ret - n, &out) == false) { logger_error("unzip_update error"); return -1; } - return (int) out.length() - saved_count; + + n = (int) out.length() - saved_count; + + // 如果新解压数据为 0,则有可能是本次解压过程需要的压缩数据不完整, + // 或有可能是此次读到了最后的 8 个字节的尾部字段所至,所以需要再 + // 尝试读一次,以期读到下一部分数据或读完所有的数据体 + if (n == 0) + goto READ_AGAIN; + + return n; } int http_client::read_request_body(string& out, bool clean, @@ -757,7 +1045,7 @@ bool http_client::body_gets(string& out, bool nonl /* = true */, size_t* size /* = NULL */) { if (buf_ == NULL) - buf_ = new string(4096); + buf_ = NEW string(4096); size_t len, size_saved = out.length(); diff --git a/lib_acl_cpp/src/http/http_header.cpp b/lib_acl_cpp/src/http/http_header.cpp index 88f715ff9..de679ed86 100644 --- a/lib_acl_cpp/src/http/http_header.cpp +++ b/lib_acl_cpp/src/http/http_header.cpp @@ -52,6 +52,7 @@ void http_header::init() range_total_ = -1; content_length_ = -1; chunked_transfer_ = false; + transfer_gzip_ = false; } void http_header::clear() @@ -805,6 +806,18 @@ bool http_header::build_response(string& out) const out << "Content-Range: bytes=" << range_from_ << '-' << range_to_ << '/' << range_total_ << "\r\n"; + // 如果是 gzip 压缩数据,当非 chunked 传输时,必须取消 Content-Length 字段, + // 同时禁止保持长连接,即: Connection: close + if (transfer_gzip_) + { + out << "Content-Encoding: gzip\r\n"; + + if (!chunked_transfer_ && keep_alive_) + const_cast(this)->keep_alive_ = false; + if (content_length_ > 0) + const_cast(this)->content_length_ = -1; + } + build_common(out); out << "\r\n"; return true; @@ -825,4 +838,12 @@ http_header& http_header::set_cgi_mode(bool on) return *this; } +http_header& http_header::set_transfer_gzip(bool on) +{ + transfer_gzip_ = on; + if (transfer_gzip_) + is_request_ = false; + return *this; +} + } // namespace acl end diff --git a/lib_acl_cpp/src/http/http_request.cpp b/lib_acl_cpp/src/http/http_request.cpp index 8e22f348f..f100d9caa 100644 --- a/lib_acl_cpp/src/http/http_request.cpp +++ b/lib_acl_cpp/src/http/http_request.cpp @@ -207,7 +207,7 @@ bool http_request::write_head() client_->reset(); // 重置状态 // 发送 HTTP 请求头 - if (client_->write_head(header_) > 0) + if (client_->write_head(header_) == true) return true; close(); @@ -274,7 +274,7 @@ bool http_request::send_request(const void* data, size_t len) client_->reset(); // 重置状态 // 写 HTTP 请求头 - if (client_->write_head(header_) < 0) + if (client_->write_head(header_) == false) { close(); return false; @@ -632,7 +632,7 @@ bool http_request::get_body(string& out, const char* to_charset /* = NULL */) return true; } -int http_request::read_body(string& out, bool clean /* = NULL */, +int http_request::read_body(string& out, bool clean /* = false */, int* real_size /* = NULL */) { if (clean) diff --git a/lib_acl_cpp/src/stdlib/zlib_stream.cpp b/lib_acl_cpp/src/stdlib/zlib_stream.cpp index 317ade8b5..ae06c6800 100644 --- a/lib_acl_cpp/src/stdlib/zlib_stream.cpp +++ b/lib_acl_cpp/src/stdlib/zlib_stream.cpp @@ -10,16 +10,19 @@ # if defined(ACL_WINDOWS) || defined(USE_DYNAMIC) typedef int (*deflateInit_fn)(z_stream*, int, const char*, int); +typedef int (*deflateInit2_fn)(z_stream*, int, int, int, int, int); typedef int (*deflate_fn)(z_stream*, int); typedef int (*deflateReset_fn)(z_stream*); typedef int (*deflateEnd_fn)(z_stream*); typedef int (*inflateInit_fn)(z_stream*, int, const char*, int); +typedef int (*inflateInit2_fn)(z_stream*, int, int, int, int, int); typedef int (*inflate_fn)(z_stream*, int); typedef int (*inflateReset_fn)(z_stream*); typedef int (*inflateEnd_fn)(z_stream*); static deflateInit_fn __deflateInit = NULL; +static deflateInit2_fn __deflateInit2 = NULL; static deflate_fn __deflate = NULL; static deflateReset_fn __deflateReset = NULL; static deflateEnd_fn __deflateEnd = NULL; @@ -61,6 +64,11 @@ static void __zlib_dll_load(void) logger_fatal("load deflateInit from zlib.dll error: %s", acl_last_serror()); + __deflateInit2 = (deflateInit2_fn) acl_dlsym(__zlib_dll, "deflateInit2"); + if (__deflateInit2 == NULL) + logger_fatal("load deflateInit from zlib.dll error: %s", + acl_last_serror()); + __deflate = (deflate_fn) acl_dlsym(__zlib_dll, "deflate"); if (__deflate == NULL) logger_fatal("load deflate from zlib.dll error: %s", @@ -101,6 +109,7 @@ static void __zlib_dll_load(void) } # else # define __deflateInit deflateInit_ +# define __deflateInit2 deflateInit2 # define __deflate deflate # define __deflateReset deflateReset # define __deflateEnd deflateEnd @@ -353,11 +362,16 @@ bool zlib_stream::flush_out(int (*func)(z_stream*, int), return (true); } -bool zlib_stream::zip_begin(zlib_level_t level /* = zlib_default */) +bool zlib_stream::zip_begin(zlib_level_t level /* = zlib_default */, + int wbits /* = zlib_wbits_15 */, + zlib_mlevel_t mlevel /* = zlib_memlevel_9 */) { is_compress_ = true; - int ret = __deflateInit(zstream_, level, - ZLIB_VERSION, sizeof(z_stream)); +// int ret = __deflateInit(zstream_, level, +// ZLIB_VERSION, sizeof(z_stream)); + + int ret = __deflateInit2(zstream_, level, Z_DEFLATED, + wbits, mlevel, Z_DEFAULT_STRATEGY); if (ret != Z_OK) { logger_error("deflateInit error");