--------------------- PatchSet 11166 Date: 2007/01/19 01:10:12 Author: hno Branch: HEAD Tag: (none) Log: Primitive support for HTTP/1.1 chunked encoding, working around broken servers sending chunked encoding in response to HTTP/1.0 requests. Members: src/HttpHeader.c:1.90->1.91 src/client_side.c:1.692->1.693 src/enums.h:1.236->1.237 src/http.c:1.418->1.419 src/structs.h:1.506->1.507 Index: squid/src/HttpHeader.c =================================================================== RCS file: /cvsroot/squid/squid/src/HttpHeader.c,v retrieving revision 1.90 retrieving revision 1.91 diff -u -r1.90 -r1.91 --- squid/src/HttpHeader.c 29 Nov 2006 05:31:48 -0000 1.90 +++ squid/src/HttpHeader.c 19 Jan 2007 01:10:12 -0000 1.91 @@ -1,6 +1,6 @@ /* - * $Id: HttpHeader.c,v 1.90 2006/11/29 05:31:48 adrian Exp $ + * $Id: HttpHeader.c,v 1.91 2007/01/19 01:10:12 hno Exp $ * * DEBUG: section 55 HTTP Header * AUTHOR: Alex Rousskov @@ -113,12 +113,14 @@ {"Retry-After", HDR_RETRY_AFTER, ftStr}, /* for now (ftDate_1123 or ftInt!) */ {"Server", HDR_SERVER, ftStr}, {"Set-Cookie", HDR_SET_COOKIE, ftStr}, - {"Title", HDR_TITLE, ftStr}, {"Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr}, + {"Te", HDR_TE, ftStr}, + {"Trailer", HDR_TRAILER, ftStr}, {"Upgrade", HDR_UPGRADE, ftStr}, /* for now */ {"User-Agent", HDR_USER_AGENT, ftStr}, {"Vary", HDR_VARY, ftStr}, /* for now */ {"Via", HDR_VIA, ftStr}, /* for now */ + {"Expect", HDR_EXPECT, ftStr}, {"Warning", HDR_WARNING, ftStr}, /* for now */ {"WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr}, {"Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr}, @@ -157,6 +159,8 @@ HDR_LINK, HDR_PRAGMA, HDR_PROXY_CONNECTION, HDR_PROXY_SUPPORT, + HDR_TE, + HDR_TRAILER, HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_VARY, @@ -165,7 +169,7 @@ HDR_WWW_AUTHENTICATE, HDR_AUTHENTICATION_INFO, HDR_PROXY_AUTHENTICATION_INFO, - /* HDR_EXPECT, HDR_TE, HDR_TRAILER */ + HDR_EXPECT, #if X_ACCELERATOR_VARY HDR_X_ACCELERATOR_VARY, #endif @@ -177,8 +181,8 @@ { HDR_CACHE_CONTROL, HDR_CONNECTION, HDR_DATE, HDR_PRAGMA, HDR_TRANSFER_ENCODING, + HDR_TRAILER, HDR_UPGRADE, - /* HDR_TRAILER, */ HDR_VIA }; @@ -216,7 +220,7 @@ HDR_IF_MATCH, HDR_IF_MODIFIED_SINCE, HDR_IF_NONE_MATCH, HDR_IF_RANGE, HDR_MAX_FORWARDS, HDR_PROXY_CONNECTION, HDR_PROXY_AUTHORIZATION, HDR_RANGE, HDR_REFERER, HDR_REQUEST_RANGE, - HDR_USER_AGENT, HDR_X_FORWARDED_FOR + HDR_USER_AGENT, HDR_X_FORWARDED_FOR, HDR_TE, HDR_EXPECT }; /* header accounting */ Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid/squid/src/client_side.c,v retrieving revision 1.692 retrieving revision 1.693 diff -u -r1.692 -r1.693 --- squid/src/client_side.c 19 Jan 2007 00:23:52 -0000 1.692 +++ squid/src/client_side.c 19 Jan 2007 01:10:12 -0000 1.693 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.692 2007/01/19 00:23:52 hno Exp $ + * $Id: client_side.c,v 1.693 2007/01/19 01:10:12 hno Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -1842,6 +1842,9 @@ /* remove Set-Cookie if a hit */ if (http->flags.hit) httpHeaderDelById(hdr, HDR_SET_COOKIE); + httpHeaderDelById(hdr, HDR_TRAILER); + httpHeaderDelById(hdr, HDR_TRANSFER_ENCODING); + httpHeaderDelById(hdr, HDR_UPGRADE); /* handle Connection header */ if (httpHeaderHas(hdr, HDR_CONNECTION)) { /* anything that matches Connection list member will be deleted */ @@ -1900,6 +1903,10 @@ } else if (http->entry->timestamp < squid_curtime) httpHeaderPutInt(hdr, HDR_AGE, squid_curtime - http->entry->timestamp); + if (!httpHeaderHas(hdr, HDR_CONTENT_LENGTH) && http->entry->mem_obj && http->entry->store_status == STORE_OK) { + rep->content_length = contentLen(http->entry); + httpHeaderPutSize(hdr, HDR_CONTENT_LENGTH, rep->content_length); + } } /* Filter unproxyable authentication types */ if (http->log_type != LOG_TCP_DENIED && @@ -1967,7 +1974,7 @@ } /* Append Via */ { - LOCAL_ARRAY(char, bbuf, MAX_URL + 32); + char bbuf[MAX_URL + 32]; String strVia = httpHeaderGetList(hdr, HDR_VIA); snprintf(bbuf, sizeof(bbuf), "%d.%d %s", rep->sline.version.major, @@ -3112,10 +3119,7 @@ } else if ((done = clientCheckTransferDone(http)) != 0 || size == 0) { debug(33, 5) ("clientWriteComplete: FD %d transfer is DONE\n", fd); /* We're finished case */ - if (httpReplyBodySize(http->request->method, entry->mem_obj->reply) < 0) { - debug(33, 5) ("clientWriteComplete: closing, content_length < 0\n"); - comm_close(fd); - } else if (!done) { + if (!done) { debug(33, 5) ("clientWriteComplete: closing, !done\n"); comm_close(fd); } else if (clientGotNotEnough(http)) { @@ -3479,6 +3483,7 @@ http->uri = xstrdup(uri); http->log_uri = xstrndup(uri, MAX_URL); http->range_iter.boundary = StringNull; + httpBuildVersion(&http->http_ver, 1, 0); dlinkAdd(http, &http->active, &ClientActiveRequests); return http; } @@ -4783,7 +4788,8 @@ { squid_off_t cl = httpReplyBodySize(http->request->method, http->entry->mem_obj->reply); int hs = http->entry->mem_obj->reply->hdr_sz; - assert(cl >= 0); + if (cl < 0) + return 0; if (http->out.offset != cl + hs) return 1; return 0; Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid/squid/src/enums.h,v retrieving revision 1.236 retrieving revision 1.237 diff -u -r1.236 -r1.237 --- squid/src/enums.h 19 Jan 2007 00:21:01 -0000 1.236 +++ squid/src/enums.h 19 Jan 2007 01:10:12 -0000 1.237 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.236 2007/01/19 00:21:01 hno Exp $ + * $Id: enums.h,v 1.237 2007/01/19 01:10:12 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -209,6 +209,9 @@ HDR_CONTENT_MD5, HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, + HDR_TE, + HDR_TRANSFER_ENCODING, + HDR_TRAILER, HDR_COOKIE, HDR_DATE, HDR_ETAG, @@ -236,12 +239,11 @@ HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE, - HDR_TITLE, - HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_USER_AGENT, HDR_VARY, HDR_VIA, + HDR_EXPECT, HDR_WARNING, HDR_WWW_AUTHENTICATE, HDR_AUTHENTICATION_INFO, Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid/squid/src/http.c,v retrieving revision 1.418 retrieving revision 1.419 diff -u -r1.418 -r1.419 --- squid/src/http.c 23 Oct 2006 21:34:17 -0000 1.418 +++ squid/src/http.c 19 Jan 2007 01:10:12 -0000 1.419 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.418 2006/10/23 21:34:17 hno Exp $ + * $Id: http.c,v 1.419 2007/01/19 01:10:12 hno Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -409,12 +409,14 @@ } /* rewrite this later using new interfaces @?@ */ -static void +static size_t httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) { StoreEntry *entry = httpState->entry; size_t hdr_len; size_t hdr_size; + size_t old_size; + size_t done; HttpReply *reply = entry->mem_obj->reply; Ctx ctx = ctx_enter(entry->mem_obj->url); debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", @@ -422,6 +424,7 @@ if (memBufIsNull(&httpState->reply_hdr)) memBufDefInit(&httpState->reply_hdr); assert(httpState->reply_hdr_state == 0); + old_size = httpState->reply_hdr.size; memBufAppend(&httpState->reply_hdr, buf, size); hdr_len = httpState->reply_hdr.size; if (hdr_len > 4 && strncmp(httpState->reply_hdr.buf, "HTTP/", 5)) { @@ -431,19 +434,19 @@ httpBuildVersion(&reply->sline.version, 0, 9); reply->sline.status = HTTP_INVALID_HEADER; ctx_exit(ctx); - return; + return 0; } hdr_size = headersEnd(httpState->reply_hdr.buf, hdr_len); if (hdr_size) hdr_len = hdr_size; if (hdr_len > Config.maxReplyHeaderSize) { debug(11, 1) ("httpProcessReplyHeader: Too large reply header\n"); - if (!memBufIsNull(&httpState->reply_hdr)) - memBufClean(&httpState->reply_hdr); + storeAppend(entry, httpState->reply_hdr.buf, httpState->reply_hdr.size); + memBufClean(&httpState->reply_hdr); reply->sline.status = HTTP_HEADER_TOO_LARGE; httpState->reply_hdr_state += 2; ctx_exit(ctx); - return; + return size; } /* headers can be incomplete only if object still arriving */ if (!hdr_size) { @@ -451,7 +454,7 @@ hdr_size = hdr_len; else { ctx_exit(ctx); - return; /* headers not complete */ + return size; /* headers not complete */ } } safe_free(entry->mem_obj->vary_headers); @@ -467,17 +470,48 @@ /* Parse headers into reply structure */ /* what happens if we fail to parse here? */ httpReplyParse(reply, httpState->reply_hdr.buf, hdr_size); + storeAppend(entry, httpState->reply_hdr.buf, hdr_size); + done = hdr_size - old_size; if (reply->sline.status >= HTTP_INVALID_HEADER) { debug(11, 3) ("httpProcessReplyHeader: Non-HTTP-compliant header: '%s'\n", httpState->reply_hdr.buf); memBufClean(&httpState->reply_hdr); ctx_exit(ctx); - return; + return done; } if (!peer_supports_connection_pinning(httpState)) httpState->orig_request->flags.no_connection_auth = 1; storeTimestampsSet(entry); /* Check if object is cacheable or not based on reply code */ debug(11, 3) ("httpProcessReplyHeader: HTTP CODE: %d\n", reply->sline.status); + if (httpHeaderHas(&reply->header, HDR_TRANSFER_ENCODING)) { + String tr = httpHeaderGetList(&reply->header, HDR_TRANSFER_ENCODING); + const char *pos = NULL; + const char *item = NULL; + int ilen = 0; + if (strListGetItem(&tr, ',', &item, &ilen, &pos)) { + if (ilen == 7 && strncasecmp(item, "chunked", ilen) == 0) { + httpState->flags.chunked = 1; + if (!strListGetItem(&tr, ',', &item, &ilen, &pos)) + item = NULL; + } + if (item) { + /* Can't handle other transfer-encodings */ + debug(11, 1) ("Unexpected transfer encoding '%s'\n", strBuf(tr)); + reply->sline.status = HTTP_INVALID_HEADER; + return done; + } + } + stringClean(&tr); + if (httpState->flags.chunked && reply->content_length >= 0) { + /* Can't have a content-length in chunked encoding */ + reply->sline.status = HTTP_INVALID_HEADER; + return done; + } + } + if (!httpState->flags.chunked) { + /* non-chunked. Handle as one single big chunk (-1 if terminated by EOF) */ + httpState->chunk_size = httpReplyBodySize(httpState->orig_request->method, reply); + } if (httpHeaderHas(&reply->header, HDR_VARY) #if X_ACCELERATOR_VARY || httpHeaderHas(&reply->header, HDR_X_ACCELERATOR_VARY) @@ -488,7 +522,7 @@ vary = httpMakeVaryMark(httpState->orig_request, reply); if (!vary) { httpMakePrivate(entry); - goto no_cache; + goto no_cache; /* XXX Would be better if this was used by the swicht statement below */ } entry->mem_obj->vary_headers = xstrdup(vary); if (strBuf(httpState->orig_request->vary_encoding)) @@ -543,36 +577,7 @@ #if HEADERS_LOG headersLog(1, 0, httpState->request->method, reply); #endif -} - -static int -httpPconnTransferDone(HttpStateData * httpState) -{ - /* return 1 if we got the last of the data on a persistent connection */ - MemObject *mem = httpState->entry->mem_obj; - HttpReply *reply = mem->reply; - squid_off_t clen; - debug(11, 3) ("httpPconnTransferDone: FD %d\n", httpState->fd); - debug(11, 5) ("httpPconnTransferDone: content_length=%" PRINTF_OFF_T "\n", - reply->content_length); - /* If we haven't seen the end of reply headers, we are not done */ - if (httpState->reply_hdr_state < 2) - return 0; - clen = httpReplyBodySize(httpState->request->method, reply); - /* If the body size is unknown we must wait for EOF */ - if (clen < 0) - return 0; - /* Barf if we got more than we asked for */ - if (mem->inmem_hi > clen + reply->hdr_sz) - return -1; - /* If there is no message body, we can be persistent */ - if (0 == clen) - return 1; - /* If the body size is known, we must wait until we've gotten all of it. */ - if (mem->inmem_hi < clen + reply->hdr_sz) - return 0; - /* We got it all */ - return 1; + return done; } /* Small helper function to verify if connection pinning is supported or not @@ -615,6 +620,187 @@ return rc; } +static void +httpAppendBody(HttpStateData * httpState, const char *buf, ssize_t len, int buffer_filled) +{ + StoreEntry *entry = httpState->entry; + const request_t *request = httpState->request; + const request_t *orig_request = httpState->orig_request; + struct in_addr *client_addr = NULL; + u_short client_port = 0; + int fd = httpState->fd; + int complete = httpState->eof; + int keep_alive = !httpState->eof; + while (len > 0) { + if (httpState->chunk_size > 0) { + size_t size = len; + if (size > httpState->chunk_size) + size = httpState->chunk_size; + httpState->chunk_size -= size; + storeAppend(httpState->entry, buf, size); + buf += size; + len -= size; + } else if (httpState->chunk_size < 0) { + /* non-chunked without content-length */ + storeAppend(httpState->entry, buf, len); + len = 0; + } else if (httpState->flags.chunked) { + char *eol = memchr(buf, '\n', len); + size_t size = eol - buf + 1; + if (!eol) + size = len; + stringAppend(&httpState->chunkhdr, buf, size); + buf += size; + len -= size; + if (eol) { + if (!httpState->flags.trailer) { + /* chunk header */ + char *end = NULL; + int badchunk = 0; + debug(11, 3) ("Chunk header '%s'\n", strBuf(httpState->chunkhdr)); + httpState->chunk_size = strto_off_t(strBuf(httpState->chunkhdr), &end, 16); + if (end == strBuf(httpState->chunkhdr)) + badchunk = 1; + while (end && (*end == '\r' || *end == ' ' || *end == '\t')) + end++; + if (httpState->chunk_size < 0 || !end || (*end != '\n' && *end == ';')) { + debug(11, 0) ("Invalid chunk header '%s'\n", strBuf(httpState->chunkhdr)); + comm_close(fd); + return; + } + if (badchunk) + continue; /* Skip blank lines */ + debug(11, 2) ("Chunk size %" PRINTF_OFF_T "\n", httpState->chunk_size); + if (httpState->chunk_size == 0) { + debug(11, 3) ("Processing trailer\n"); + httpState->flags.trailer = 1; + } + } else { + /* trailer */ + const char *p = strBuf(httpState->chunkhdr); + while (*p == '\r') + p++; + if (*p == '\n') { + complete = 1; + debug(11, 2) ("Chunked response complete\n"); + } + } + stringReset(&httpState->chunkhdr, NULL); + } + } else { + /* Don't know what to do with this data. Bail out */ + break; + } + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + /* + * the above storeAppend() call could ABORT this entry, + * in that case, the server FD should already be closed. + * there's nothing for us to do. + */ + return; + } + } + if (!httpState->chunk_size && !httpState->flags.chunked) + complete = 1; + if (!complete && len == 0) { + /* Wait for more data or EOF condition */ + if (httpState->flags.keepalive_broken) { + commSetTimeout(fd, 10, NULL, NULL); + } else { + commSetTimeout(fd, Config.Timeout.read, NULL, NULL); + } + commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); + return; + } + /* Is it a incomplete reply? */ + if (httpState->chunk_size > 0) { + debug(11, 1) ("Short response on port %d. Expecting %" PRINTF_OFF_T " octets more\n", comm_local_port(fd), httpState->chunk_size); + comm_close(fd); + return; + } + /* + * Verify that the connection is clean + */ + if (len == 0 && buffer_filled >= 0) { + char buf2[4]; + statCounter.syscalls.sock.reads++; + len = FD_READ_METHOD(fd, buf2, sizeof(buf2)); + if ((len < 0 && !ignoreErrno(errno)) || len == 0) { + keep_alive = 0; + } + } + if (len > 0) { + debug(11, Config.onoff.relaxed_header_parser <= 0 || keep_alive ? 1 : 2) + ("httpReadReply: Excess data from \"%s %s\"\n", + RequestMethodStr[orig_request->method], + storeUrl(entry)); + comm_close(fd); + return; + } + /* + * Verified and done with the reply + */ + fwdComplete(httpState->fwd); + + /* + * If we didn't send a keep-alive request header, then this + * can not be a persistent connection. + */ + if (!httpState->flags.keepalive) + keep_alive = 0; + /* + * If we haven't sent the whole request then this can not be a persistent + * connection. + */ + if (!httpState->flags.request_sent) { + debug(11, 1) ("httpReadReply: Request not yet fully sent \"%s %s\"\n", + RequestMethodStr[orig_request->method], + storeUrl(entry)); + keep_alive = 0; + } + /* + * What does the reply have to say about keep-alive? + */ + if (!entry->mem_obj->reply->keep_alive) + keep_alive = 0; + if (keep_alive) { + int pinned = 0; +#if LINUX_TPROXY + if (orig_request->flags.tproxy) { + client_addr = &httpState->request->client_addr; + } +#endif + /* yes we have to clear all these! */ + commSetDefer(fd, NULL, NULL); + commSetTimeout(fd, -1, NULL, NULL); + commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); +#if DELAY_POOLS + delayClearNoDelay(fd); +#endif + comm_remove_close_handler(fd, httpStateFree, httpState); + fwdUnregister(fd, httpState->fwd); + if (request->flags.pinned) { + pinned = 1; + } else if (request->flags.connection_auth && request->flags.auth_sent) { + pinned = 1; + } + if (orig_request->pinned_connection && pinned) { + clientPinConnection(orig_request->pinned_connection, fd, orig_request, httpState->peer, request->flags.connection_auth); + } else if (httpState->peer) { + if (httpState->peer->options.originserver) + pconnPush(fd, httpState->peer->name, httpState->peer->http_port, httpState->orig_request->host, client_addr, client_port); + else + pconnPush(fd, httpState->peer->name, httpState->peer->http_port, NULL, client_addr, client_port); + } else { + pconnPush(fd, request->host, request->port, NULL, client_addr, client_port); + } + httpState->fd = -1; + httpStateFree(fd, httpState); + } else { + comm_close(fd); + } +} + /* This will be called when data is ready to be read from fd. Read until * error or connection closed. */ /* XXX this function is too long! */ @@ -622,19 +808,17 @@ httpReadReply(int fd, void *data) { HttpStateData *httpState = data; - LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF); + LOCAL_ARRAY(char, buf, SQUID_TCP_SO_RCVBUF + 1); StoreEntry *entry = httpState->entry; - const request_t *request = httpState->request; - const request_t *orig_request = httpState->orig_request; - int len; + ssize_t len; int bin; int clen; + int done = 0; size_t read_sz = SQUID_TCP_SO_RCVBUF; - struct in_addr *client_addr = NULL; - u_short client_port = 0; #if DELAY_POOLS delay_id delay_id; #endif + int buffer_filled; if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { comm_close(fd); @@ -651,7 +835,8 @@ errno = 0; statCounter.syscalls.sock.reads++; len = FD_READ_METHOD(fd, buf, read_sz); - debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, len); + buffer_filled = len == read_sz; + debug(11, 5) ("httpReadReply: FD %d: len %d.\n", fd, (int) len); if (len > 0) { fd_bytes(fd, len, FD_READ); #if DELAY_POOLS @@ -663,6 +848,7 @@ for (clen = len - 1, bin = 0; clen; bin++) clen >>= 1; IOStats.Http.read_hist[bin]++; + buf[len] = '\0'; } if (!httpState->reply_hdr.size && len > 0 && fd_table[fd].uses > 1) { /* Skip whitespace */ @@ -711,13 +897,14 @@ fwdFail(httpState->fwd, errorCon(ERR_INVALID_RESP, HTTP_BAD_GATEWAY, httpState->fwd->request)); httpState->fwd->flags.dont_retry = 1; } else { - fwdComplete(httpState->fwd); + httpAppendBody(httpState, NULL, 0, -1); /* EOF */ + return; } comm_close(fd); return; } else { if (httpState->reply_hdr_state < 2) { - httpProcessReplyHeader(httpState, buf, len); + done = httpProcessReplyHeader(httpState, buf, len); if (httpState->reply_hdr_state == 2) { http_status s = entry->mem_obj->reply->sline.status; if (s == HTTP_HEADER_TOO_LARGE) { @@ -744,121 +931,14 @@ */ if (!fwdReforwardableStatus(s)) EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); - } - } - storeAppend(entry, buf, len); - if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { - /* - * the above storeAppend() call could ABORT this entry, - * in that case, the server FD should already be closed. - * there's nothing for us to do. - */ - return; - } - switch (httpPconnTransferDone(httpState)) { - case 1: - { - int keep_alive = 1; - /* - * If we didn't send a keep-alive request header, then this - * can not be a persistent connection. - */ - if (!httpState->flags.keepalive) - keep_alive = 0; - /* - * If we haven't sent the whole request then this can not be a persistent - * connection. - */ - if (!httpState->flags.request_sent) { - debug(11, 1) ("httpReadReply: Request not yet fully sent \"%s %s\"\n", - RequestMethodStr[orig_request->method], - storeUrl(entry)); - keep_alive = 0; - } - /* - * What does the reply have to say about keep-alive? - */ - if (!entry->mem_obj->reply->keep_alive) - keep_alive = 0; - /* - * Verify that the connection is clean - */ - if (len == read_sz) { - statCounter.syscalls.sock.reads++; - len = FD_READ_METHOD(fd, buf, SQUID_TCP_SO_RCVBUF); - if ((len < 0 && !ignoreErrno(errno)) || len == 0) { - keep_alive = 0; - } else if (len > 0) { - debug(11, Config.onoff.relaxed_header_parser <= 0 || keep_alive ? 1 : 2) - ("httpReadReply: Excess data from \"%s %s\"\n", - RequestMethodStr[orig_request->method], - storeUrl(entry)); - storeAppend(entry, buf, len); - keep_alive = 0; - } - } - if (keep_alive) { - int pinned = 0; -#if LINUX_TPROXY - if (orig_request->flags.tproxy) { - client_addr = &httpState->request->client_addr; - } -#endif - /* yes we have to clear all these! */ - commSetDefer(fd, NULL, NULL); - commSetTimeout(fd, -1, NULL, NULL); - commSetSelect(fd, COMM_SELECT_READ, NULL, NULL, 0); -#if DELAY_POOLS - delayClearNoDelay(fd); -#endif - comm_remove_close_handler(fd, httpStateFree, httpState); - fwdUnregister(fd, httpState->fwd); - if (request->flags.pinned) { - pinned = 1; - } else if (request->flags.connection_auth && request->flags.auth_sent) { - pinned = 1; - } - if (orig_request->pinned_connection && pinned) { - clientPinConnection(orig_request->pinned_connection, fd, orig_request, httpState->peer, request->flags.connection_auth); - } else if (httpState->peer) { - if (httpState->peer->options.originserver) - pconnPush(fd, httpState->peer->name, httpState->peer->http_port, httpState->orig_request->host, client_addr, client_port); - else - pconnPush(fd, httpState->peer->name, httpState->peer->http_port, NULL, client_addr, client_port); - } else { - pconnPush(fd, request->host, request->port, NULL, client_addr, client_port); - } - fwdComplete(httpState->fwd); - httpState->fd = -1; - httpStateFree(fd, httpState); - } else { - fwdComplete(httpState->fwd); - comm_close(fd); - } - } - return; - case 0: - /* Wait for more data or EOF condition */ - if (httpState->flags.keepalive_broken) { - commSetTimeout(fd, 10, NULL, NULL); } else { commSetTimeout(fd, Config.Timeout.read, NULL, NULL); + commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); + return; } - commSetSelect(fd, COMM_SELECT_READ, httpReadReply, httpState, 0); - return; - case -1: - /* Server is nasty on us. Shut down */ - debug(11, Config.onoff.relaxed_header_parser <= 0 || entry->mem_obj->reply->keep_alive ? 1 : 2) - ("httpReadReply: Excess data from \"%s %s\"\n", - RequestMethodStr[orig_request->method], - storeUrl(entry)); - fwdComplete(httpState->fwd); - comm_close(fd); - return; - default: - fatal("Unexpected httpPconnTransferDone() status\n"); - break; } + httpAppendBody(httpState, buf + done, len - done, buffer_filled); + return; } } @@ -970,7 +1050,7 @@ debug(11, 5) ("httpBuildRequestHeader: %s: %s\n", strBuf(e->name), strBuf(e->value)); if (!httpRequestHdrAllowed(e, &strConnection)) { - debug(11, 2) ("'%s' header denied by anonymize_headers configuration\n", + debug(11, 2) ("'%s' header is a hop-by-hop connections header\n", strBuf(e->name)); continue; } @@ -1059,8 +1139,18 @@ if (!Config.onoff.via) httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); break; - case HDR_PROXY_CONNECTION: case HDR_CONNECTION: + case HDR_KEEP_ALIVE: + /* case HDR_PROXY_AUTHORIZATION: is special and handled above */ + case HDR_PROXY_AUTHENTICATE: + case HDR_TE: + case HDR_TRAILER: + case HDR_TRANSFER_ENCODING: + case HDR_UPGRADE: + case HDR_PROXY_CONNECTION: + case HDR_EXPECT: + /* hop-by-hop headers. Don't forward */ + break; case HDR_CACHE_CONTROL: /* append these after the loop if needed */ break; Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid/squid/src/structs.h,v retrieving revision 1.506 retrieving revision 1.507 diff -u -r1.506 -r1.507 --- squid/src/structs.h 29 Nov 2006 15:58:52 -0000 1.506 +++ squid/src/structs.h 19 Jan 2007 01:10:12 -0000 1.507 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.506 2006/11/29 15:58:52 adrian Exp $ + * $Id: structs.h,v 1.507 2007/01/19 01:10:12 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1093,6 +1093,8 @@ unsigned int request_sent:1; unsigned int front_end_https:2; unsigned int originpeer:1; + unsigned int chunked:1; + unsigned int trailer:1; }; struct _HttpStateData { @@ -1108,6 +1110,8 @@ FwdState *fwd; char *body_buf; int body_buf_sz; + squid_off_t chunk_size; + String chunkhdr; }; struct _icpUdpData {