ripe-atlas-fw: imported version 45504550

Signed-off-by: Bjørn Mork <bjorn@mork.no>

1 files changed, 228 insertions, 36 deletions

diff --git a/eperd/httpget.c b/eperd/httpget.c
index c380f18..d0d3e4c 100644
--- a/eperd/httpget.c
+++ b/eperd/httpget.c
@@ -19,7 +19,7 @@
 #define SAFE_PREFIX_IN ATLAS_DATA_OUT
 #define SAFE_PREFIX_OUT ATLAS_DATA_NEW
 
-#define CONN_TO		   5
+#define CONN_TO		   5000		/* Timeout in milliseconds */
 
 #define ENV2STATE(env) \
 	((struct hgstate *)((char *)env - offsetof(struct hgstate, tu_env)))
@@ -39,17 +39,21 @@ static struct option longopts[]=
 	{ "post-file",	required_argument, NULL, 'p' },
 	{ "post-header", required_argument, NULL, 'h' },
 	{ "post-footer", required_argument, NULL, 'f' },
+	{ "read-limit",	required_argument, NULL, 'r' },
 	{ "store-headers", required_argument, NULL, 'H' },
 	{ "store-body",	required_argument, NULL, 'B' },
 	{ "user-agent",	required_argument, NULL, 'u' },
+	{ "timeout",	required_argument, NULL, 'S' },
+	{ "etim",	no_argument, NULL, 't' },
+	{ "eetim",	no_argument, NULL, 'T' },
 	{ NULL, }
 };
 
-enum readstate { READ_STATUS, READ_HEADER, READ_BODY, READ_SIMPLE,
+enum readstate { READ_FIRST, READ_STATUS, READ_HEADER, READ_BODY, READ_SIMPLE,
 	READ_CHUNKED, READ_CHUNK_BODY, READ_CHUNK_END, READ_CHUNKED_TRAILER,
 	READ_DONE };
-enum writestate { WRITE_HEADER, WRITE_POST_HEADER, WRITE_POST_FILE,
-	WRITE_POST_FOOTER, WRITE_DONE };
+enum writestate { WRITE_FIRST, WRITE_HEADER, WRITE_POST_HEADER,
+	WRITE_POST_FILE, WRITE_POST_FOOTER, WRITE_DONE };
 
 struct hgbase
 {
@@ -84,6 +88,9 @@ struct hgstate
 	char *post_footer;
 	int max_headers;
 	int max_body;
+	int etim;
+	size_t read_limit;
+	unsigned timeout;
 
 	/* State */
 	char busy;
@@ -111,6 +118,13 @@ struct hgstate
 	time_t gstart;
 	struct timeval start;
 	double resptime;
+	double ttr;		/* Time to resolve */
+	double ttc;		/* Time to connect */
+	double ttfb;		/* Time to first byte */
+	int roffset;
+	int report_roffset;
+	int first_connect;
+	int read_truncated;
 	FILE *post_fh;
 	char *post_buf;
 
@@ -131,6 +145,10 @@ struct hgstate
 	char *result;
 	size_t reslen;
 	size_t resmax;
+
+	char *result2;
+	size_t reslen2;
+	size_t resmax2;
 };
 
 static struct hgbase *hg_base;
@@ -138,6 +156,7 @@ static struct hgbase *hg_base;
 static void report(struct hgstate *state);
 static void add_str(struct hgstate *state, const char *str);
 static void add_str_quoted(struct hgstate *state, char *str);
+static void add_str2(struct hgstate *state, const char *str);
 
 static struct hgbase *httpget_base_new(struct event_base *event_base)
 {
@@ -304,6 +323,7 @@ static void timeout_callback(int __attribute((unused)) unused,
 	}
 	switch(state->readstate)
 	{
+	case READ_FIRST:
 	case READ_STATUS:
 		add_str(state, DBQ(err) ":" DBQ(timeout reading status) ", ");
 		report(state);
@@ -342,11 +362,13 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[],
 {
 	int c, i, do_combine, do_get, do_head, do_post,
 		max_headers, max_body, only_v4, only_v6,
-		do_all, do_http10;
-	size_t newsiz;
+		do_all, do_http10, do_etim, do_eetim;
+	size_t newsiz, read_limit;
+	unsigned timeout;
 	char *url, *check;
 	char *post_file, *output_file, *post_footer, *post_header,
-		*A_arg, *store_headers, *store_body;
+		*A_arg, *store_headers, *store_body, *read_limit_str,
+		*timeout_str;
 	const char *user_agent;
 	char *host, *port, *hostport, *path;
 	struct hgstate *state;
@@ -365,9 +387,13 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[],
 	output_file= NULL;
 	store_headers= NULL;
 	store_body= NULL;
+	read_limit_str= NULL;
+	timeout_str= NULL;
 	A_arg= NULL;
 	only_v4= 0;
 	only_v6= 0;
+	do_etim= 0;
+	do_eetim= 0;
 	user_agent= "httpget for atlas.ripe.net";
 
 	if (!hg_base)
@@ -390,55 +416,67 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[],
 		case '1':
 			do_http10= 0;
 			break;
-		case 'a':				/* --all */
-			do_all= 1;
+		case '4':
+			only_v4= 1;
+			only_v6= 0;
+			break;
+		case '6':
+			only_v6= 1;
+			only_v4= 0;
 			break;
 		case 'A':
 			A_arg= optarg;
 			break;
+		case 'a':				/* --all */
+			do_all= 1;
+			break;
+		case 'B':				/* --store-body */
+			store_body= optarg;
+			break;
 		case 'c':				/* --combine */
 			do_combine= 1;
 			break;
-		case 'O':
-			output_file= optarg;
+		case 'E':				/* --head */
+			do_get = 0;
+			do_head = 1;
+			do_post = 0;
 			break;
 		case 'g':				/* --get */
 			do_get = 1;
 			do_head = 0;
 			do_post = 0;
 			break;
-		case 'E':				/* --head */
-			do_get = 0;
-			do_head = 1;
-			do_post = 0;
+		case 'f':				/* --post-footer */
+			post_footer= optarg;
 			break;
-		case 'P':				/* --post */
-			do_get = 0;
-			do_head = 0;
-			do_post = 1;
+		case 'H':				/* --store-headers */
+			store_headers= optarg;
 			break;
 		case 'h':				/* --post-header */
 			post_header= optarg;
 			break;
-		case 'f':				/* --post-footer */
-			post_footer= optarg;
+		case 'O':
+			output_file= optarg;
+			break;
+		case 'P':				/* --post */
+			do_get = 0;
+			do_head = 0;
+			do_post = 1;
 			break;
 		case 'p':				/* --post-file */
 			post_file= optarg;
 			break;
-		case 'H':				/* --store-headers */
-			store_headers= optarg;
+		case 'r':
+			read_limit_str= optarg;		/* --read-limit */
 			break;
-		case 'B':				/* --store-body */
-			store_body= optarg;
+		case 'T':				/* --etim */
+			do_eetim= 1;
 			break;
-		case '4':
-			only_v4= 1;
-			only_v6= 0;
+		case 't':				/* --etim */
+			do_etim= 1;
 			break;
-		case '6':
-			only_v6= 1;
-			only_v4= 0;
+		case 'S':
+			timeout_str= optarg;		/* --timeout */
 			break;
 		case 'u':				/* --user-agent */
 			user_agent= optarg;
@@ -472,6 +510,16 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[],
 		}
 		fclose(fh);
 	}
+
+	if (A_arg)
+	{
+		if (!validate_atlas_id(A_arg))
+		{
+			crondlog(LVL8 "bad atlas ID '%s'", A_arg);
+			return NULL;
+		}
+	}
+
 	if (post_header && !validate_filename(post_header, SAFE_PREFIX_IN))
 	{
 		crondlog(LVL8 "insecure file '%s'", post_header);
@@ -515,6 +563,32 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[],
 		}
 	}
 
+	read_limit= 0;
+	if (read_limit_str)
+	{
+		read_limit= strtoul(read_limit_str, &check, 10);
+		if (check[0] != '\0')
+		{
+			crondlog(LVL8
+				"unable to parse argument (--read-limit) '%s'",
+				read_limit_str);
+			return NULL;
+		}
+	}
+
+	timeout= CONN_TO;
+	if (timeout_str)
+	{
+		timeout= strtoul(timeout_str, &check, 10);
+		if (check[0] != '\0')
+		{
+			crondlog(LVL8
+				"unable to parse argument (--timeout) '%s'",
+				timeout_str);
+			return NULL;
+		}
+	}
+
 	if (!parse_url(url, &host, &port, &hostport, &path))
 	{
 		/* Do we need to report an error? */
@@ -546,11 +620,21 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[],
 	state->user_agent= user_agent ? strdup(user_agent) : NULL;
 	state->max_headers= max_headers;
 	state->max_body= max_body;
+	state->read_limit= read_limit;
+	state->timeout= timeout;
 
 	state->only_v4= 2;
 
 	state->only_v4= !!only_v4;	/* Gcc bug? */
 	state->only_v6= !!only_v6;
+	
+	if (do_eetim)
+		state->etim= 2;
+	else if (do_etim)
+		state->etim= 1;
+	else
+		state->etim= 0;
+
 
 	//evtimer_assign(&state->timer, state->base->event_base,
 	//	timeout_callback, state);
@@ -649,12 +733,26 @@ static void report(struct hgstate *state)
 			state->sin6.sin6_family == AF_INET6 ? 6 : 4);
 		add_str(state, line);
 
+		if (state->read_truncated)
+			add_str(state, ", " DBQ(read-truncated) ": True");
+
 		getnameinfo((struct sockaddr *)&state->sin6, state->socklen,
 			namebuf, sizeof(namebuf), NULL, 0, NI_NUMERICHOST);
 
 		snprintf(line, sizeof(line), ", " DBQ(dst_addr) ":" DBQ(%s),
 			namebuf);
 		add_str(state, line);
+
+		/* End of readtiming */
+		if (state->etim >= 2)
+		{
+			add_str2(state, " ]");
+			add_str(state, state->result2);
+			free(state->result2);
+			state->result2= NULL;
+			state->resmax2= 0;
+			state->reslen2= 0;
+		}
 	}
 
 	if (!state->connecting && !state->dnserr)
@@ -684,6 +782,17 @@ static void report(struct hgstate *state)
 			state->headers_size,
 			state->content_offset);
 		add_str(state, line);
+		if (state->etim >= 1)
+		{
+			snprintf(line, sizeof(line),
+				", " DBQ(ttr) ":%f"
+				", " DBQ(ttc) ":%f"
+				", " DBQ(ttfb) ":%f",
+				state->ttr,
+				state->ttc,
+				state->ttfb);
+			add_str(state, line);
+		}
 	}
 
 	if (!state->dnserr)
@@ -726,14 +835,17 @@ static void report(struct hgstate *state)
 
 	tu_cleanup(&state->tu_env);
 
+	state->busy= 0;
 	if (state->base->done)
 		state->base->done(state);
-	state->busy= 0;
 }
 
 static int get_input(struct hgstate *state)
 {
 	int n;
+	double t;
+	struct timeval endtime;
+	char line[80];
 
 	/* Assume that we always end up with a full buffer anyway */
 	if (state->linemax == 0)
@@ -764,12 +876,27 @@ static int get_input(struct hgstate *state)
 		return -1;	/* We cannot get more data */
 	}
 
+	if (state->etim >= 2 && state->report_roffset)
+	{
+		gettimeofday(&endtime, NULL);
+		t= (endtime.tv_sec-state->start.tv_sec)*1e3 +
+			(endtime.tv_usec-state->start.tv_usec)/1e3;
+		if (state->roffset != 0)
+			add_str2(state, ",");
+		snprintf(line, sizeof(line),
+			" { " DBQ(o) ":" DBQ(%d)
+			", " DBQ(t) ": %f }", state->roffset, t);
+		add_str2(state, line);
+		state->report_roffset= 0;
+	}
+
 	n= bufferevent_read(state->bev,
 		&state->line[state->linelen],
 		state->linemax-state->linelen);
 	if (n < 0)
 		return -1;
 	state->linelen += n;
+	state->roffset += n;
 	return 0;
 }
 
@@ -823,9 +950,24 @@ static void add_str_quoted(struct hgstate *state, char *str)
 	}
 }
 
+static void add_str2(struct hgstate *state, const char *str)
+{
+	size_t len;
+
+	len= strlen(str);
+	if (state->reslen2 + len+1 > state->resmax2)
+	{
+		state->resmax2= 2*state->reslen2 + len+1 + 80;
+		state->result2= xrealloc(state->result2, state->resmax2);
+	}
+	memcpy(state->result2+state->reslen2, str, len+1);
+	state->reslen2 += len;
+}
+
 static void err_status(struct hgstate *state, const char *reason)
 {
 	char line[80];
+
 	snprintf(line, sizeof(line),
 		DBQ(err) ":" DBQ(bad status line: %s) ", ", 
 		reason);
@@ -836,6 +978,7 @@ static void err_status(struct hgstate *state, const char *reason)
 static void err_header(struct hgstate *state, const char *reason)
 {
 	char line[80];
+
 	if (state->max_headers != 0)
 		add_str(state, " ], ");
 	snprintf(line, sizeof(line),
@@ -864,10 +1007,39 @@ static void readcb(struct bufferevent *bev UNUSED_PARAM, void *ptr)
 
 	state= ENV2STATE(ptr);
 
+	state->report_roffset= 1;
 	for (;;)
 	{
+		if (state->read_limit > 0 &&
+			state->roffset >= state->read_limit)
+		{
+			state->read_truncated= 1;
+			switch(state->readstate)
+			{
+			case READ_FIRST:
+			case READ_STATUS:
+				err_status(state, "read truncated");
+				return;
+			case READ_HEADER:
+				err_header(state, "read truncated");
+				return;
+			default:
+				state->readstate= READ_DONE;
+				break;
+			}
+		}
 		switch(state->readstate)
 		{
+		case READ_FIRST:
+			gettimeofday(&endtime, NULL);
+			state->ttfb= (endtime.tv_sec-
+				state->start.tv_sec)*1e3 +
+				(endtime.tv_usec-state->start.tv_usec)/1e3;
+			state->readstate= READ_STATUS;
+			state->roffset= 0;
+			if (state->etim >= 2)
+				add_str2(state, ", " DBQ(readtiming) ": [");
+			continue;
 		case READ_STATUS:
 		case READ_HEADER:
 		case READ_CHUNKED:
@@ -1425,6 +1597,7 @@ static void writecb(struct bufferevent *bev, void *ptr)
 	struct evbuffer *output;
 	off_t cLength;
 	struct stat sb;
+	struct timeval endtime;
 
 	state= ENV2STATE(ptr);
 
@@ -1432,6 +1605,13 @@ static void writecb(struct bufferevent *bev, void *ptr)
 	{
 		switch(state->writestate)
 		{
+		case WRITE_FIRST:
+			gettimeofday(&endtime, NULL);
+			state->ttc= (endtime.tv_sec-
+				state->start.tv_sec)*1e3 +
+				(endtime.tv_usec-state->start.tv_usec)/1e3;
+			state->writestate= WRITE_HEADER;
+			continue;
 		case WRITE_HEADER:
 			output= bufferevent_get_output(bev);
 			evbuffer_add_printf(output, "%s %s HTTP/1.%c\r\n",
@@ -1583,6 +1763,7 @@ static void beforeconnect(struct tu_env *env,
 	struct sockaddr *addr, socklen_t addrlen)
 {
 	struct hgstate *state;
+	struct timeval endtime;
 
 	state= ENV2STATE(env);
 
@@ -1590,13 +1771,15 @@ static void beforeconnect(struct tu_env *env,
 	memcpy(&state->sin6, addr, state->socklen);
 
 	state->connecting= 1;
-	state->readstate= READ_STATUS;
-	state->writestate= WRITE_HEADER;
+	state->readstate= READ_FIRST;
+	state->writestate= WRITE_FIRST;
 
 	state->linelen= 0;
 	state->lineoffset= 0;
 	state->headers_size= 0;
 	state->tot_headers= 0;
+	state->roffset= 0;
+	state->read_truncated= 0;
 
 	/* Clear result */
 	if (!state->do_all || !state->do_combine)
@@ -1604,6 +1787,13 @@ static void beforeconnect(struct tu_env *env,
 
 	add_str(state, "{ ");
 
+	if (state->first_connect)
+	{
+		gettimeofday(&endtime, NULL);
+		state->ttr= (endtime.tv_sec-state->start.tv_sec)*1e3 +
+			(endtime.tv_usec-state->start.tv_usec)/1e3;
+		state->first_connect= 0;
+	}
 	gettimeofday(&state->start, NULL);
 }
 
@@ -1688,6 +1878,8 @@ static void httpget_start(void *state)
 	hgstate->readstate= READ_STATUS;
 	hgstate->writestate= WRITE_HEADER;
 	hgstate->gstart= time(NULL);
+	gettimeofday(&hgstate->start, NULL);
+	hgstate->first_connect= 1;
 
 	memset(&hints, '\0', sizeof(hints));
 	hints.ai_socktype= SOCK_STREAM;
@@ -1695,8 +1887,8 @@ static void httpget_start(void *state)
 		hints.ai_family= AF_INET;
 	else if (hgstate->only_v6)
 		hints.ai_family= AF_INET6;
-	interval.tv_sec= CONN_TO;
-	interval.tv_usec= 0;
+	interval.tv_sec= hgstate->timeout / 1000;
+	interval.tv_usec= (hgstate->timeout % 1000) * 1000;
 	tu_connect_to_name(&hgstate->tu_env, hgstate->host, hgstate->port,
 		&interval, &hints, timeout_callback,
 		reporterr, dnscount, beforeconnect,