diff options
Diffstat (limited to 'eperd/httpget.c')
-rw-r--r-- | eperd/httpget.c | 264 |
1 files changed, 228 insertions, 36 deletions
diff --git a/eperd/httpget.c b/eperd/httpget.c index c380f18..d0d3e4c 100644 --- a/eperd/httpget.c +++ b/eperd/httpget.c @@ -19,7 +19,7 @@ #define SAFE_PREFIX_IN ATLAS_DATA_OUT #define SAFE_PREFIX_OUT ATLAS_DATA_NEW -#define CONN_TO 5 +#define CONN_TO 5000 /* Timeout in milliseconds */ #define ENV2STATE(env) \ ((struct hgstate *)((char *)env - offsetof(struct hgstate, tu_env))) @@ -39,17 +39,21 @@ static struct option longopts[]= { "post-file", required_argument, NULL, 'p' }, { "post-header", required_argument, NULL, 'h' }, { "post-footer", required_argument, NULL, 'f' }, + { "read-limit", required_argument, NULL, 'r' }, { "store-headers", required_argument, NULL, 'H' }, { "store-body", required_argument, NULL, 'B' }, { "user-agent", required_argument, NULL, 'u' }, + { "timeout", required_argument, NULL, 'S' }, + { "etim", no_argument, NULL, 't' }, + { "eetim", no_argument, NULL, 'T' }, { NULL, } }; -enum readstate { READ_STATUS, READ_HEADER, READ_BODY, READ_SIMPLE, +enum readstate { READ_FIRST, READ_STATUS, READ_HEADER, READ_BODY, READ_SIMPLE, READ_CHUNKED, READ_CHUNK_BODY, READ_CHUNK_END, READ_CHUNKED_TRAILER, READ_DONE }; -enum writestate { WRITE_HEADER, WRITE_POST_HEADER, WRITE_POST_FILE, - WRITE_POST_FOOTER, WRITE_DONE }; +enum writestate { WRITE_FIRST, WRITE_HEADER, WRITE_POST_HEADER, + WRITE_POST_FILE, WRITE_POST_FOOTER, WRITE_DONE }; struct hgbase { @@ -84,6 +88,9 @@ struct hgstate char *post_footer; int max_headers; int max_body; + int etim; + size_t read_limit; + unsigned timeout; /* State */ char busy; @@ -111,6 +118,13 @@ struct hgstate time_t gstart; struct timeval start; double resptime; + double ttr; /* Time to resolve */ + double ttc; /* Time to connect */ + double ttfb; /* Time to first byte */ + int roffset; + int report_roffset; + int first_connect; + int read_truncated; FILE *post_fh; char *post_buf; @@ -131,6 +145,10 @@ struct hgstate char *result; size_t reslen; size_t resmax; + + char *result2; + size_t reslen2; + size_t resmax2; }; static struct hgbase *hg_base; @@ -138,6 +156,7 @@ static struct hgbase *hg_base; static void report(struct hgstate *state); static void add_str(struct hgstate *state, const char *str); static void add_str_quoted(struct hgstate *state, char *str); +static void add_str2(struct hgstate *state, const char *str); static struct hgbase *httpget_base_new(struct event_base *event_base) { @@ -304,6 +323,7 @@ static void timeout_callback(int __attribute((unused)) unused, } switch(state->readstate) { + case READ_FIRST: case READ_STATUS: add_str(state, DBQ(err) ":" DBQ(timeout reading status) ", "); report(state); @@ -342,11 +362,13 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[], { int c, i, do_combine, do_get, do_head, do_post, max_headers, max_body, only_v4, only_v6, - do_all, do_http10; - size_t newsiz; + do_all, do_http10, do_etim, do_eetim; + size_t newsiz, read_limit; + unsigned timeout; char *url, *check; char *post_file, *output_file, *post_footer, *post_header, - *A_arg, *store_headers, *store_body; + *A_arg, *store_headers, *store_body, *read_limit_str, + *timeout_str; const char *user_agent; char *host, *port, *hostport, *path; struct hgstate *state; @@ -365,9 +387,13 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[], output_file= NULL; store_headers= NULL; store_body= NULL; + read_limit_str= NULL; + timeout_str= NULL; A_arg= NULL; only_v4= 0; only_v6= 0; + do_etim= 0; + do_eetim= 0; user_agent= "httpget for atlas.ripe.net"; if (!hg_base) @@ -390,55 +416,67 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[], case '1': do_http10= 0; break; - case 'a': /* --all */ - do_all= 1; + case '4': + only_v4= 1; + only_v6= 0; + break; + case '6': + only_v6= 1; + only_v4= 0; break; case 'A': A_arg= optarg; break; + case 'a': /* --all */ + do_all= 1; + break; + case 'B': /* --store-body */ + store_body= optarg; + break; case 'c': /* --combine */ do_combine= 1; break; - case 'O': - output_file= optarg; + case 'E': /* --head */ + do_get = 0; + do_head = 1; + do_post = 0; break; case 'g': /* --get */ do_get = 1; do_head = 0; do_post = 0; break; - case 'E': /* --head */ - do_get = 0; - do_head = 1; - do_post = 0; + case 'f': /* --post-footer */ + post_footer= optarg; break; - case 'P': /* --post */ - do_get = 0; - do_head = 0; - do_post = 1; + case 'H': /* --store-headers */ + store_headers= optarg; break; case 'h': /* --post-header */ post_header= optarg; break; - case 'f': /* --post-footer */ - post_footer= optarg; + case 'O': + output_file= optarg; + break; + case 'P': /* --post */ + do_get = 0; + do_head = 0; + do_post = 1; break; case 'p': /* --post-file */ post_file= optarg; break; - case 'H': /* --store-headers */ - store_headers= optarg; + case 'r': + read_limit_str= optarg; /* --read-limit */ break; - case 'B': /* --store-body */ - store_body= optarg; + case 'T': /* --etim */ + do_eetim= 1; break; - case '4': - only_v4= 1; - only_v6= 0; + case 't': /* --etim */ + do_etim= 1; break; - case '6': - only_v6= 1; - only_v4= 0; + case 'S': + timeout_str= optarg; /* --timeout */ break; case 'u': /* --user-agent */ user_agent= optarg; @@ -472,6 +510,16 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[], } fclose(fh); } + + if (A_arg) + { + if (!validate_atlas_id(A_arg)) + { + crondlog(LVL8 "bad atlas ID '%s'", A_arg); + return NULL; + } + } + if (post_header && !validate_filename(post_header, SAFE_PREFIX_IN)) { crondlog(LVL8 "insecure file '%s'", post_header); @@ -515,6 +563,32 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[], } } + read_limit= 0; + if (read_limit_str) + { + read_limit= strtoul(read_limit_str, &check, 10); + if (check[0] != '\0') + { + crondlog(LVL8 + "unable to parse argument (--read-limit) '%s'", + read_limit_str); + return NULL; + } + } + + timeout= CONN_TO; + if (timeout_str) + { + timeout= strtoul(timeout_str, &check, 10); + if (check[0] != '\0') + { + crondlog(LVL8 + "unable to parse argument (--timeout) '%s'", + timeout_str); + return NULL; + } + } + if (!parse_url(url, &host, &port, &hostport, &path)) { /* Do we need to report an error? */ @@ -546,11 +620,21 @@ static void *httpget_init(int __attribute((unused)) argc, char *argv[], state->user_agent= user_agent ? strdup(user_agent) : NULL; state->max_headers= max_headers; state->max_body= max_body; + state->read_limit= read_limit; + state->timeout= timeout; state->only_v4= 2; state->only_v4= !!only_v4; /* Gcc bug? */ state->only_v6= !!only_v6; + + if (do_eetim) + state->etim= 2; + else if (do_etim) + state->etim= 1; + else + state->etim= 0; + //evtimer_assign(&state->timer, state->base->event_base, // timeout_callback, state); @@ -649,12 +733,26 @@ static void report(struct hgstate *state) state->sin6.sin6_family == AF_INET6 ? 6 : 4); add_str(state, line); + if (state->read_truncated) + add_str(state, ", " DBQ(read-truncated) ": True"); + getnameinfo((struct sockaddr *)&state->sin6, state->socklen, namebuf, sizeof(namebuf), NULL, 0, NI_NUMERICHOST); snprintf(line, sizeof(line), ", " DBQ(dst_addr) ":" DBQ(%s), namebuf); add_str(state, line); + + /* End of readtiming */ + if (state->etim >= 2) + { + add_str2(state, " ]"); + add_str(state, state->result2); + free(state->result2); + state->result2= NULL; + state->resmax2= 0; + state->reslen2= 0; + } } if (!state->connecting && !state->dnserr) @@ -684,6 +782,17 @@ static void report(struct hgstate *state) state->headers_size, state->content_offset); add_str(state, line); + if (state->etim >= 1) + { + snprintf(line, sizeof(line), + ", " DBQ(ttr) ":%f" + ", " DBQ(ttc) ":%f" + ", " DBQ(ttfb) ":%f", + state->ttr, + state->ttc, + state->ttfb); + add_str(state, line); + } } if (!state->dnserr) @@ -726,14 +835,17 @@ static void report(struct hgstate *state) tu_cleanup(&state->tu_env); + state->busy= 0; if (state->base->done) state->base->done(state); - state->busy= 0; } static int get_input(struct hgstate *state) { int n; + double t; + struct timeval endtime; + char line[80]; /* Assume that we always end up with a full buffer anyway */ if (state->linemax == 0) @@ -764,12 +876,27 @@ static int get_input(struct hgstate *state) return -1; /* We cannot get more data */ } + if (state->etim >= 2 && state->report_roffset) + { + gettimeofday(&endtime, NULL); + t= (endtime.tv_sec-state->start.tv_sec)*1e3 + + (endtime.tv_usec-state->start.tv_usec)/1e3; + if (state->roffset != 0) + add_str2(state, ","); + snprintf(line, sizeof(line), + " { " DBQ(o) ":" DBQ(%d) + ", " DBQ(t) ": %f }", state->roffset, t); + add_str2(state, line); + state->report_roffset= 0; + } + n= bufferevent_read(state->bev, &state->line[state->linelen], state->linemax-state->linelen); if (n < 0) return -1; state->linelen += n; + state->roffset += n; return 0; } @@ -823,9 +950,24 @@ static void add_str_quoted(struct hgstate *state, char *str) } } +static void add_str2(struct hgstate *state, const char *str) +{ + size_t len; + + len= strlen(str); + if (state->reslen2 + len+1 > state->resmax2) + { + state->resmax2= 2*state->reslen2 + len+1 + 80; + state->result2= xrealloc(state->result2, state->resmax2); + } + memcpy(state->result2+state->reslen2, str, len+1); + state->reslen2 += len; +} + static void err_status(struct hgstate *state, const char *reason) { char line[80]; + snprintf(line, sizeof(line), DBQ(err) ":" DBQ(bad status line: %s) ", ", reason); @@ -836,6 +978,7 @@ static void err_status(struct hgstate *state, const char *reason) static void err_header(struct hgstate *state, const char *reason) { char line[80]; + if (state->max_headers != 0) add_str(state, " ], "); snprintf(line, sizeof(line), @@ -864,10 +1007,39 @@ static void readcb(struct bufferevent *bev UNUSED_PARAM, void *ptr) state= ENV2STATE(ptr); + state->report_roffset= 1; for (;;) { + if (state->read_limit > 0 && + state->roffset >= state->read_limit) + { + state->read_truncated= 1; + switch(state->readstate) + { + case READ_FIRST: + case READ_STATUS: + err_status(state, "read truncated"); + return; + case READ_HEADER: + err_header(state, "read truncated"); + return; + default: + state->readstate= READ_DONE; + break; + } + } switch(state->readstate) { + case READ_FIRST: + gettimeofday(&endtime, NULL); + state->ttfb= (endtime.tv_sec- + state->start.tv_sec)*1e3 + + (endtime.tv_usec-state->start.tv_usec)/1e3; + state->readstate= READ_STATUS; + state->roffset= 0; + if (state->etim >= 2) + add_str2(state, ", " DBQ(readtiming) ": ["); + continue; case READ_STATUS: case READ_HEADER: case READ_CHUNKED: @@ -1425,6 +1597,7 @@ static void writecb(struct bufferevent *bev, void *ptr) struct evbuffer *output; off_t cLength; struct stat sb; + struct timeval endtime; state= ENV2STATE(ptr); @@ -1432,6 +1605,13 @@ static void writecb(struct bufferevent *bev, void *ptr) { switch(state->writestate) { + case WRITE_FIRST: + gettimeofday(&endtime, NULL); + state->ttc= (endtime.tv_sec- + state->start.tv_sec)*1e3 + + (endtime.tv_usec-state->start.tv_usec)/1e3; + state->writestate= WRITE_HEADER; + continue; case WRITE_HEADER: output= bufferevent_get_output(bev); evbuffer_add_printf(output, "%s %s HTTP/1.%c\r\n", @@ -1583,6 +1763,7 @@ static void beforeconnect(struct tu_env *env, struct sockaddr *addr, socklen_t addrlen) { struct hgstate *state; + struct timeval endtime; state= ENV2STATE(env); @@ -1590,13 +1771,15 @@ static void beforeconnect(struct tu_env *env, memcpy(&state->sin6, addr, state->socklen); state->connecting= 1; - state->readstate= READ_STATUS; - state->writestate= WRITE_HEADER; + state->readstate= READ_FIRST; + state->writestate= WRITE_FIRST; state->linelen= 0; state->lineoffset= 0; state->headers_size= 0; state->tot_headers= 0; + state->roffset= 0; + state->read_truncated= 0; /* Clear result */ if (!state->do_all || !state->do_combine) @@ -1604,6 +1787,13 @@ static void beforeconnect(struct tu_env *env, add_str(state, "{ "); + if (state->first_connect) + { + gettimeofday(&endtime, NULL); + state->ttr= (endtime.tv_sec-state->start.tv_sec)*1e3 + + (endtime.tv_usec-state->start.tv_usec)/1e3; + state->first_connect= 0; + } gettimeofday(&state->start, NULL); } @@ -1688,6 +1878,8 @@ static void httpget_start(void *state) hgstate->readstate= READ_STATUS; hgstate->writestate= WRITE_HEADER; hgstate->gstart= time(NULL); + gettimeofday(&hgstate->start, NULL); + hgstate->first_connect= 1; memset(&hints, '\0', sizeof(hints)); hints.ai_socktype= SOCK_STREAM; @@ -1695,8 +1887,8 @@ static void httpget_start(void *state) hints.ai_family= AF_INET; else if (hgstate->only_v6) hints.ai_family= AF_INET6; - interval.tv_sec= CONN_TO; - interval.tv_usec= 0; + interval.tv_sec= hgstate->timeout / 1000; + interval.tv_usec= (hgstate->timeout % 1000) * 1000; tu_connect_to_name(&hgstate->tu_env, hgstate->host, hgstate->port, &interval, &hints, timeout_callback, reporterr, dnscount, beforeconnect, |