Skip to content

Commit

Permalink
update to svn add tcp output and some modules
Browse files Browse the repository at this point in the history
  • Loading branch information
kongjian committed Feb 5, 2015
1 parent adf1f92 commit 2ee1a94
Show file tree
Hide file tree
Showing 13 changed files with 981 additions and 6 deletions.
6 changes: 6 additions & 0 deletions conf/tsar.conf
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ mod_tcprt off
mod_proc off pidname
mod_pharos off
mod_tmd4 off
#mod_erpc on /etc/tsar/erpc.conf

####output_interface file,db,nagios
output_interface file
Expand All @@ -47,6 +48,11 @@ output_stdio_mod mod_swap,mod_partition,mod_cpu,mod_mem,mod_lvs,mod_haproxy,mod_
#output_db_mod mod_swap,mod_partition,mod_cpu,mod_mem,mod_traffic,mod_load,mod_tcp,mod_udp,mod_pcsw,mod_io
#output_db_addr console2:56677

####[output_tcp]
#output_tcp_mod mod_swap,mod_cpu
#output_tcp_addr localhost:9666
#output_tcp_merge on

####support include other mod conf
include /etc/tsar/conf.d/*.conf

Expand Down
5 changes: 5 additions & 0 deletions include/config.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ struct configure {
char output_db_mod[LEN_512]; /* which mod will output */
char output_db_addr[LEN_512]; /* db addr */

/* output tcp */
char output_tcp_mod[LEN_512];
char output_tcp_addr[LEN_256];
char output_tcp_merge[LEN_256];

/* output nagios */
char server_addr[LEN_512];
int server_port;
Expand Down
32 changes: 32 additions & 0 deletions include/output_tcp.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@

/*
* (C) 2010-2011 Alibaba Group Holding Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/


#ifndef TSAR_OUTPUT_TCP_H
#define TSAR_OUTPUT_TCP_H


#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>


void output_tcp(int have_collect);
struct sockaddr_in * str2sa(char *);
#endif
1 change: 1 addition & 0 deletions include/tsar.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
#include "output_file.h"
#include "output_print.h"
#include "output_db.h"
#include "output_tcp.h"
#include "output_nagios.h"
#include "common.h"

Expand Down
64 changes: 64 additions & 0 deletions info.md
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,70 @@ top=10 #指定最多采集的域名个数,按照请求总个数排列
domain=a.com b.com #指定特定需要采集的域名列表,分隔符为空格,逗号,或者制表符
在/etc/tsar/tsar.conf中指定配置文件的路径:mod_nginx_domain on /tmp/my.conf

####nginx_domain_traffic
nginx配置是:

req_status_zone server "$host" 20M;
req_status server;

# req_status_zone_add_indecator 指令:可以在req status输出的每一行最后添加新的字段
# 这里添加的字段用于统计nginx的变量: $2xx_bytes_sent, $3xx_bytes_sent, $4xx_bytes_sent, $5xx_bytes_sent
# $2xx_bytes_sent: 请求返回2xx时候,发送给客户端的数据量(如果请求非2xx则该变量为0)
req_status_zone_add_indecator server $2xx_bytes_sent $3xx_bytes_sent $4xx_bytes_sent $5xx_bytes_sent;

location /traffic_status {
req_status_show;
}

输出实例:

module004033.sqa.cm4 tsar $ tsar --nginx_domain_traffic -li1
Time -----------------localhost:8080----------------- ----------------www.foo.com:8080----------------
Time bytin bytout 2XXout 3XXout 4XXout 5XXout bytin bytout 2XXout 3XXout 4XXout 5XXout
09/01/15-13:45:48 0.00 0.00 0.00 0.00 0.00 0.00 410.1K 16.6M 16.6M 0.00 0.00 0.00
09/01/15-13:45:49 0.00 0.00 0.00 0.00 0.00 0.00 407.8K 16.5M 16.5M 0.00 0.00 0.00
09/01/15-13:45:51 159.0K 287.4K 0.00 0.00 0.00 287.4K 258.6K 10.5M 10.5M 0.00 0.00 0.00
09/01/15-13:45:52 245.5K 443.5K 0.00 0.00 0.00 443.5K 224.2K 9.1M 9.1M 0.00 0.00 0.00

字段含义:
* bytin: 收到的请求字节数byte/s
* bytout: 输出的应答字节数byte/s
* 2XXout: 输出的2XX应答字节数byte/s
* 3XXout: 输出的3XX应答字节数byte/s
* 4XXout: 输出的4XX应答字节数byte/s
* 5XXout: 输出的5XX应答字节数byte/s

####nginx_ups
用于输出nginx upstream想关信息
nginx配置是:

req_status_zone server "$host" 20M;
req_status server;
req_status_zone_add_indecator server $response_fbt_time $upstream_response_fbt_time $upstream_response_length;

location /traffic_status {
req_status_show;
}

输出实例:

module004033.sqa.cm4 tsar $ tsar --nginx_ups -li1
Time ----------------------------nginx_ups---------------------------
Time traff qps 4XX 5XX rqps rt fbt ufbt
09/01/15-16:26:29 15.8M 3.9K 3.9K 0.00 0.00 9.7K 9.7K 9.7K
09/01/15-16:26:30 15.8M 3.9K 3.9K 0.00 0.00 9.7K 9.7K 9.7K
09/01/15-16:26:31 4.9M 1.2K 1.2K 0.00 0.00 3.0K 3.0K 3.0K

字段含义:
* traff: 后端返回的应答body的流量(不包括http应答头部)
* qps: 后端qps
* rqps: 后端总qps(包含重试的qps + 后端qps)
* 4XX: 后端返回4XX状态码的qps
* 5XX: 后端返回5XX状态码的qps
* rt: 后端应答时间
* fbt: tengine首字节时间
* ufbt: 后端应答首字节时间

###squid
####字段含义
* qps: 每秒请求数
Expand Down
8 changes: 4 additions & 4 deletions modules/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,28 +10,28 @@ ifeq ($(UNAME_S),Darwin)
OBJS = mod_swap.so mod_cpu.so mod_mem.so mod_lvs.so mod_haproxy.so \
mod_traffic.so mod_pernic.so mod_squid.so mod_load.so mod_tcp.so mod_udp.so mod_tcpx.so mod_proc.so\
mod_apache.so mod_pcsw.so mod_nginx.so mod_nginx_code.so mod_cgblkio.so \
mod_cgcpu.so mod_cgmem.so mod_ncpu.so mod_rndc.so \
mod_cgcpu.so mod_cgmem.so mod_ncpu.so mod_rndc.so mod_erpc.so \
mod_ts_cache.so mod_ts_client.so mod_ts_err.so mod_ts_os.so \
mod_ts_storage.so mod_ts_conn.so mod_ts_codes.so\
mod_swift.so mod_swift_code.so mod_swift_store.so mod_swift_spdy.so\
mod_swift_fwd.so mod_swift_domain.so mod_swift_swapdir.so \
mod_swift_purge.so mod_swift_tcmalloc.so mod_swift_sys.so \
mod_swift_esi.so mod_swift_balancer.so mod_swift_blc_fwd.so mod_swift_conn.so\
mod_tmd.so mod_percpu.so mod_rpi.so mod_tcprt.so mod_shell.so mod_nginx_domain.so mod_pharos.so mod_pharos_load.so\
mod_tmd4.so
mod_tmd4.so mod_nginx_domain_traffic.so mod_nginx_ups.so
else
OBJS = mod_swap.so mod_partition.so mod_cpu.so mod_mem.so mod_lvs.so mod_haproxy.so \
mod_traffic.so mod_pernic.so mod_squid.so mod_load.so mod_tcp.so mod_udp.so mod_tcpx.so mod_proc.so\
mod_apache.so mod_pcsw.so mod_io.so mod_nginx.so mod_nginx_code.so mod_cgblkio.so \
mod_cgcpu.so mod_cgmem.so mod_ncpu.so mod_rndc.so \
mod_cgcpu.so mod_cgmem.so mod_ncpu.so mod_rndc.so mod_erpc.so \
mod_ts_cache.so mod_ts_client.so mod_ts_err.so mod_ts_os.so \
mod_ts_storage.so mod_ts_conn.so mod_ts_codes.so\
mod_swift.so mod_swift_code.so mod_swift_store.so mod_swift_spdy.so\
mod_swift_fwd.so mod_swift_domain.so mod_swift_swapdir.so \
mod_swift_purge.so mod_swift_tcmalloc.so mod_swift_sys.so \
mod_swift_esi.so mod_swift_balancer.so mod_swift_blc_fwd.so mod_swift_conn.so\
mod_tmd.so mod_percpu.so mod_rpi.so mod_tcprt.so mod_shell.so mod_nginx_domain.so mod_pharos.so mod_pharos_load.so\
mod_tmd4.so
mod_tmd4.so mod_nginx_domain_traffic.so mod_nginx_ups.so
endif

all: $(OBJS)
Expand Down
190 changes: 190 additions & 0 deletions modules/mod_erpc.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
#define _GNU_SOURCE

#include <sys/socket.h>
#include <sys/un.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include "tsar.h"
#include <stdio.h>

#define MAX 16
int method_num = 0;

struct stats_erpc {
int port;
char name [256]; /* method name */
unsigned long long rt; /* total bytes in */
unsigned long long reqc; /* total bytes out */
unsigned long long reqs; /* total connections */
unsigned long long rspc; /* total requests */
unsigned long long rsps; /* 2XX status code */
};

/* struct for http domain */
static struct stats_erpc erpc_stats[MAX];

static char *erpc_usage = " --erpc erpc statistics";

static struct mod_info erpc_info[] = {
{" rt", SUMMARY_BIT, MERGE_SUM, STATS_NULL},
{"reqqps", SUMMARY_BIT, MERGE_SUM, STATS_NULL},
{" reqsz", SUMMARY_BIT, MERGE_SUM, STATS_NULL},
{"rspqps", SUMMARY_BIT, MERGE_SUM, STATS_NULL},
{" rspsz", SUMMARY_BIT, MERGE_SUM, STATS_NULL},
};

static void erpc_init(char *parameter)
{
FILE *fp;
int port;
char *p, *pp, *line = NULL;
size_t size = LEN_1024;
ssize_t ret = 0;

method_num = 0;
memset(erpc_stats, 0, MAX * sizeof(struct stats_erpc));

fp = fopen(parameter, "r");
if (fp == NULL) {
return;
}

while ((ret = getline(&line, &size, fp)) > 0) {
*(line + strlen(line) - 1) = '\0';
if ((p = strchr(line, '=')) == NULL) {
continue;
}
port = atoi(line);
*p++ = '\0';
pp = p;
while ((p = strchr(p, ',')) != NULL) {
*p++ = '\0';
if (method_num >= MAX) {
method_num = MAX;
break;
}
erpc_stats[method_num].port = port;
strcpy(erpc_stats[method_num].name, pp);
method_num++;
pp = p;
}
if (method_num < MAX) {
erpc_stats[method_num].port = port;
strcpy(erpc_stats[method_num].name, pp);
method_num++;
}
}

if (line != NULL) {
free(line);
}
}

static void
set_erpc_record(struct module *mod, double st_array[],
U_64 pre_array[], U_64 cur_array[], int inter)
{
int i;
if (cur_array[0] >= pre_array[0] && cur_array[3] > pre_array[3]) {
st_array[0] = (cur_array[0] - pre_array[0]) / (cur_array[3] - pre_array[3]) / 1000;
} else {
st_array[0] = 0;
}
for (i = 1; i < 5; i++) {
if (cur_array[i] >= pre_array[i]) {
st_array[i] = (cur_array[i] - pre_array[i]) * 1.0 / inter;
} else {
st_array[i] = 0;
}
}
}

void
read_erpc_stats(struct module *mod, char *parameter)
{
int i, addr_len, m, sockfd, send, pos = 0, mark = 0;
char buf[LEN_10240], request[LEN_4096], line[LEN_4096];
void *addr;
FILE *stream = NULL;
struct sockaddr_in servaddr;
//struct stats_erpc stat;

erpc_init(parameter);

addr = &servaddr;
addr_len = sizeof(servaddr);
bzero(&servaddr, sizeof(servaddr));
servaddr.sin_family = AF_INET;
inet_pton(AF_INET, "127.0.0.1", &servaddr.sin_addr);

for (i = 0; i < method_num; i++) {
servaddr.sin_port = htons(erpc_stats[i].port);
/* send request */
if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
return;
}

sprintf(request,
"GET %s HTTP/1.0\r\n"
"Host: %s\r\n"
"Accept:*/*\r\n"
"Connection: Close\r\n\r\n",
"/monitor", "127.0.0.1");

if ((m = connect(sockfd, (struct sockaddr *) addr, addr_len)) == -1 ) {
return;
}

if ((send = write(sockfd, request, strlen(request))) == -1) {
return;
}

/* read & parse request */
if ((stream = fdopen(sockfd, "r")) == NULL) {
close(sockfd);
return;
}
mark = 0;
while (fgets(line, LEN_4096, stream) != NULL) {
if (strstr(line, "methodName") != NULL) {
if (strstr(line, erpc_stats[i].name)) {
mark = 1;
} else {
mark = 0;
}
}
if (mark == 1) {
if(!strncmp(line, "processTime(us)", 15)) {
sscanf(line + 16, "%llu", &erpc_stats[i].rt);
} else if (!strncmp(line, "requestCount", 12)) {
sscanf(line + 13, "%llu", &erpc_stats[i].reqc);
} else if (!strncmp(line, "requestSize(bytes)", 18)) {
sscanf(line + 19, "%llu", &erpc_stats[i].reqs);
} else if (!strncmp(line, "responseCount", 13)) {
sscanf(line + 14, "%llu", &erpc_stats[i].rspc);
} else if (!strncmp(line, "responseSize(bytes)", 19)) {
sscanf(line + 20, "%llu", &erpc_stats[i].rsps);
}
}
}
fclose(stream);
close(sockfd);
}

for (i = 0; i < method_num; i++) {
pos += snprintf(buf + pos, LEN_10240 - pos, "%d_%s=%lld,%lld,%lld,%lld,%lld" ITEM_SPLIT, erpc_stats[i].port, erpc_stats[i].name, erpc_stats[i].rt, erpc_stats[i].reqc, erpc_stats[i].reqs, erpc_stats[i].rspc, erpc_stats[i].rsps);

if (strlen(buf) == LEN_10240 - 1) {
return;
}
}

set_mod_record(mod, buf);
}

void
mod_register(struct module *mod)
{
register_mod_fileds(mod, "--erpc", erpc_usage, erpc_info, 5, read_erpc_stats, set_erpc_record);
}
Loading

0 comments on commit 2ee1a94

Please sign in to comment.