--------------------- PatchSet 12036 Date: 2008/04/08 22:52:59 Author: hno Branch: SQUID_2_7 Tag: (none) Log: Clean up the other store clients to handle >4K headers this changes all store clients to refer to the already parsed header and then seek over the header data rather than trying to parse it again. This change is also fixing Bug #2296 in Squid-2.6 even if the problem there is different.. Members: src/errormap.c:1.3->1.3.2.1 src/net_db.c:1.176.2.1->1.176.2.2 src/peer_digest.c:1.95->1.95.2.1 src/peer_monitor.c:1.5.2.1->1.5.2.2 src/urn.c:1.82->1.82.2.1 Index: squid/src/errormap.c =================================================================== RCS file: /cvsroot/squid/squid/src/errormap.c,v retrieving revision 1.3 retrieving revision 1.3.2.1 diff -u -r1.3 -r1.3.2.1 --- squid/src/errormap.c 21 Jan 2007 12:53:58 -0000 1.3 +++ squid/src/errormap.c 8 Apr 2008 22:52:59 -0000 1.3.2.1 @@ -1,6 +1,6 @@ /* - * $Id: errormap.c,v 1.3 2007/01/21 12:53:58 adrian Exp $ + * $Id: errormap.c,v 1.3.2.1 2008/04/08 22:52:59 hno Exp $ * * DEBUG: section ?? Error Beautifier * AUTHOR: Henrik Nordstrom @@ -113,7 +113,8 @@ errorMapFetchHeaders(void *data, char *buf, ssize_t size) { ErrorMapState *state = data; - size_t hdr_size; + HttpReply *reply; + http_status status; if (EBIT_TEST(state->e->flags, ENTRY_ABORTED)) goto abort; @@ -122,27 +123,16 @@ if (!cbdataValid(state->callback_data)) goto abort; - if ((hdr_size = headersEnd(buf, size))) { - http_status status; - /* httpReplyParse(reply, buf, hdr_size); */ - HttpReply *reply = state->e->mem_obj->reply; - assert(reply); - status = reply->sline.status; - if (status != HTTP_OK) - goto abort; - /* Send object to caller (cbdataValid verified above) */ - state->callback(state->e, hdr_size, httpHeaderGetSize(&reply->header, HDR_CONTENT_LENGTH), state->callback_data); - errorMapFetchComplete(state); - goto done; - } - if (size >= 4096) { - /* Not enought space for reply headers */ + reply = state->e->mem_obj->reply; + + status = reply->sline.status; + if (status != HTTP_OK) goto abort; - } - /* Need more data */ - storeClientCopy(state->sc, state->e, size, 0, 4096, state->buf, errorMapFetchHeaders, state); - done: + /* Send object to caller (cbdataValid verified above) */ + state->callback(state->e, reply->hdr_sz, httpHeaderGetSize(&reply->header, HDR_CONTENT_LENGTH), state->callback_data); + errorMapFetchComplete(state); return; + abort: errorMapFetchAbort(state); return; Index: squid/src/net_db.c =================================================================== RCS file: /cvsroot/squid/squid/src/net_db.c,v retrieving revision 1.176.2.1 retrieving revision 1.176.2.2 diff -u -r1.176.2.1 -r1.176.2.2 --- squid/src/net_db.c 2 Jan 2008 15:50:39 -0000 1.176.2.1 +++ squid/src/net_db.c 8 Apr 2008 22:52:59 -0000 1.176.2.2 @@ -1,6 +1,6 @@ /* - * $Id: net_db.c,v 1.176.2.1 2008/01/02 15:50:39 hno Exp $ + * $Id: net_db.c,v 1.176.2.2 2008/04/08 22:52:59 hno Exp $ * * DEBUG: section 38 Network Measurement Database * AUTHOR: Duane Wessels @@ -560,29 +560,21 @@ p = buf; if (0 == ex->used) { /* skip reply headers */ - if ((hdr_sz = headersEnd(p, size))) { - debug(38, 5) ("netdbExchangeHandleReply: hdr_sz = %ld\n", (long int) hdr_sz); - rep = ex->e->mem_obj->reply; - if (0 == rep->sline.status) - httpReplyParse(rep, buf, hdr_sz); - debug(38, 3) ("netdbExchangeHandleReply: reply status %d\n", - rep->sline.status); - if (HTTP_OK != rep->sline.status) { - netdbExchangeDone(ex); - return; - } - assert(size >= hdr_sz); - ex->used += hdr_sz; + rep = ex->e->mem_obj->reply; + hdr_sz = rep->hdr_sz; + debug(38, 5) ("netdbExchangeHandleReply: hdr_sz = %ld\n", (long int) hdr_sz); + debug(38, 3) ("netdbExchangeHandleReply: reply status %d\n", + rep->sline.status); + if (HTTP_OK != rep->sline.status) { + netdbExchangeDone(ex); + return; + } + ex->used += hdr_sz; + if (size < hdr_sz) { size -= hdr_sz; p += hdr_sz; } else { - if (size >= ex->buf_sz) { - debug(38, 3) ("netdbExchangeHandleReply: Too big HTTP header, aborting\n"); - netdbExchangeDone(ex); - return; - } else { - size = 0; - } + size = 0; } } debug(38, 5) ("netdbExchangeHandleReply: start parsing loop, size = %ld\n", Index: squid/src/peer_digest.c =================================================================== RCS file: /cvsroot/squid/squid/src/peer_digest.c,v retrieving revision 1.95 retrieving revision 1.95.2.1 diff -u -r1.95 -r1.95.2.1 --- squid/src/peer_digest.c 21 Jan 2007 12:54:00 -0000 1.95 +++ squid/src/peer_digest.c 8 Apr 2008 22:52:59 -0000 1.95.2.1 @@ -1,6 +1,6 @@ /* - * $Id: peer_digest.c,v 1.95 2007/01/21 12:54:00 adrian Exp $ + * $Id: peer_digest.c,v 1.95.2.1 2008/04/08 22:52:59 hno Exp $ * * DEBUG: section 72 Peer Digest Routines * AUTHOR: Alex Rousskov @@ -334,77 +334,67 @@ peerDigestFetchReply, fetch); } -/* wait for full http headers to be received then parse them */ +/* wait for full http headers to be received */ static void peerDigestFetchReply(void *data, char *buf, ssize_t size) { DigestFetchState *fetch = data; PeerDigest *pd = fetch->pd; - size_t hdr_size; + http_status status; + HttpReply *reply; assert(pd && buf); assert(!fetch->offset); if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestFetchReply")) return; - if ((hdr_size = headersEnd(buf, size))) { - http_status status; - HttpReply *reply = fetch->entry->mem_obj->reply; - assert(reply); - httpReplyParse(reply, buf, hdr_size); - status = reply->sline.status; - debug(72, 3) ("peerDigestFetchReply: %s status: %d, expires: %ld (%+d)\n", - strBuf(pd->host), status, - (long int) reply->expires, (int) (reply->expires - squid_curtime)); - - /* this "if" is based on clientHandleIMSReply() */ - if (status == HTTP_NOT_MODIFIED) { - request_t *r = NULL; - /* our old entry is fine */ - assert(fetch->old_entry); - if (!fetch->old_entry->mem_obj->request) - fetch->old_entry->mem_obj->request = r = - requestLink(fetch->entry->mem_obj->request); - assert(fetch->old_entry->mem_obj->request); - httpReplyUpdateOnNotModified(fetch->old_entry->mem_obj->reply, reply); - storeTimestampsSet(fetch->old_entry); - /* get rid of 304 reply */ - storeClientUnregister(fetch->sc, fetch->entry, fetch); - storeUnlockObject(fetch->entry); - fetch->entry = fetch->old_entry; + reply = fetch->entry->mem_obj->reply; + assert(reply); + status = reply->sline.status; + debug(72, 3) ("peerDigestFetchReply: %s status: %d, expires: %ld (%+d)\n", + strBuf(pd->host), status, + (long int) reply->expires, (int) (reply->expires - squid_curtime)); + + /* this "if" is based on clientHandleIMSReply() */ + if (status == HTTP_NOT_MODIFIED) { + request_t *r = NULL; + /* our old entry is fine */ + assert(fetch->old_entry); + if (!fetch->old_entry->mem_obj->request) + fetch->old_entry->mem_obj->request = r = + requestLink(fetch->entry->mem_obj->request); + assert(fetch->old_entry->mem_obj->request); + httpReplyUpdateOnNotModified(fetch->old_entry->mem_obj->reply, reply); + storeTimestampsSet(fetch->old_entry); + /* get rid of 304 reply */ + storeClientUnregister(fetch->sc, fetch->entry, fetch); + storeUnlockObject(fetch->entry); + fetch->entry = fetch->old_entry; + fetch->old_entry = NULL; + /* preserve request -- we need its size to update counters */ + /* requestUnlink(r); */ + /* fetch->entry->mem_obj->request = NULL; */ + } else if (status == HTTP_OK) { + /* get rid of old entry if any */ + if (fetch->old_entry) { + debug(72, 3) ("peerDigestFetchReply: got new digest, releasing old one\n"); + storeClientUnregister(fetch->old_sc, fetch->old_entry, fetch); + storeReleaseRequest(fetch->old_entry); + storeUnlockObject(fetch->old_entry); fetch->old_entry = NULL; - /* preserve request -- we need its size to update counters */ - /* requestUnlink(r); */ - /* fetch->entry->mem_obj->request = NULL; */ - } else if (status == HTTP_OK) { - /* get rid of old entry if any */ - if (fetch->old_entry) { - debug(72, 3) ("peerDigestFetchReply: got new digest, releasing old one\n"); - storeClientUnregister(fetch->old_sc, fetch->old_entry, fetch); - storeReleaseRequest(fetch->old_entry); - storeUnlockObject(fetch->old_entry); - fetch->old_entry = NULL; - } - } else { - /* some kind of a bug */ - peerDigestFetchAbort(fetch, buf, httpStatusLineReason(&reply->sline)); - return; } - /* must have a ready-to-use store entry if we got here */ - /* can we stay with the old in-memory digest? */ - if (status == HTTP_NOT_MODIFIED && fetch->pd->cd) - peerDigestFetchStop(fetch, buf, "Not modified"); - else - storeClientCopy(fetch->sc, fetch->entry, /* have to swap in */ - 0, 0, SM_PAGE_SIZE, buf, peerDigestSwapInHeaders, fetch); } else { - /* need more data, do we have space? */ - if (size >= SM_PAGE_SIZE) - peerDigestFetchAbort(fetch, buf, "reply header too big"); - else - storeClientCopy(fetch->sc, fetch->entry, size, 0, SM_PAGE_SIZE, buf, - peerDigestFetchReply, fetch); + /* some kind of a bug */ + peerDigestFetchAbort(fetch, buf, httpStatusLineReason(&reply->sline)); + return; } + /* must have a ready-to-use store entry if we got here */ + /* can we stay with the old in-memory digest? */ + if (status == HTTP_NOT_MODIFIED && fetch->pd->cd) + peerDigestFetchStop(fetch, buf, "Not modified"); + else + storeClientCopy(fetch->sc, fetch->entry, /* have to swap in */ + 0, 0, SM_PAGE_SIZE, buf, peerDigestSwapInHeaders, fetch); } /* fetch headers from disk, pass on to SwapInCBlock */ @@ -412,34 +402,22 @@ peerDigestSwapInHeaders(void *data, char *buf, ssize_t size) { DigestFetchState *fetch = data; - size_t hdr_size; if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestSwapInHeaders")) return; assert(!fetch->offset); - if ((hdr_size = headersEnd(buf, size))) { - assert(fetch->entry->mem_obj->reply); - if (!fetch->entry->mem_obj->reply->sline.status) - httpReplyParse(fetch->entry->mem_obj->reply, buf, hdr_size); - if (fetch->entry->mem_obj->reply->sline.status != HTTP_OK) { - debug(72, 1) ("peerDigestSwapInHeaders: %s status %d got cached!\n", - strBuf(fetch->pd->host), fetch->entry->mem_obj->reply->sline.status); - peerDigestFetchAbort(fetch, buf, "internal status error"); - return; - } - fetch->offset += hdr_size; - storeClientCopy(fetch->sc, fetch->entry, size, fetch->offset, - SM_PAGE_SIZE, buf, - peerDigestSwapInCBlock, fetch); - } else { - /* need more data, do we have space? */ - if (size >= SM_PAGE_SIZE) - peerDigestFetchAbort(fetch, buf, "stored header too big"); - else - storeClientCopy(fetch->sc, fetch->entry, size, 0, SM_PAGE_SIZE, buf, - peerDigestSwapInHeaders, fetch); + assert(fetch->entry->mem_obj->reply); + if (fetch->entry->mem_obj->reply->sline.status != HTTP_OK) { + debug(72, 1) ("peerDigestSwapInHeaders: %s status %d got cached!\n", + strBuf(fetch->pd->host), fetch->entry->mem_obj->reply->sline.status); + peerDigestFetchAbort(fetch, buf, "internal status error"); + return; } + fetch->offset += fetch->entry->mem_obj->reply->hdr_sz; + storeClientCopy(fetch->sc, fetch->entry, size, fetch->offset, + SM_PAGE_SIZE, buf, + peerDigestSwapInCBlock, fetch); } static void Index: squid/src/peer_monitor.c =================================================================== RCS file: /cvsroot/squid/squid/src/peer_monitor.c,v retrieving revision 1.5.2.1 retrieving revision 1.5.2.2 diff -u -r1.5.2.1 -r1.5.2.2 --- squid/src/peer_monitor.c 23 Jan 2008 10:26:00 -0000 1.5.2.1 +++ squid/src/peer_monitor.c 8 Apr 2008 22:52:59 -0000 1.5.2.2 @@ -1,6 +1,6 @@ /* - * $Id: peer_monitor.c,v 1.5.2.1 2008/01/23 10:26:00 hno Exp $ + * $Id: peer_monitor.c,v 1.5.2.2 2008/04/08 22:52:59 hno Exp $ * * DEBUG: section ?? Peer monitoring * AUTHOR: Henrik Nordstrom @@ -47,7 +47,6 @@ store_client *sc; int size; http_status status; - int hdr_size; int offset; char *buf; int timeout_set; @@ -87,7 +86,8 @@ peerMonitorFetchReplyHeaders(void *data, char *buf, ssize_t size) { PeerMonitor *pm = data; - size_t hdr_size; + HttpReply *reply; + http_status status; if (EBIT_TEST(pm->running.e->flags, ENTRY_ABORTED)) goto completed; @@ -96,25 +96,20 @@ if (!cbdataValid(pm->peer)) goto completed; - if ((hdr_size = headersEnd(buf, size))) { - http_status status; - HttpReply *reply = pm->running.e->mem_obj->reply; - assert(reply); - /* httpReplyParse(reply, buf, hdr_size); */ - status = reply->sline.status; - pm->running.status = status; - if (status != HTTP_OK) - goto completed; - pm->running.size = size - hdr_size; + reply = pm->running.e->mem_obj->reply; + assert(reply); + status = reply->sline.status; + pm->running.status = status; + if (status != HTTP_OK) + goto completed; + if (size > reply->hdr_sz) { + pm->running.size = size - reply->hdr_sz; pm->running.offset = size; - storeClientCopy(pm->running.sc, pm->running.e, pm->running.offset, pm->running.offset, 4096, buf, peerMonitorFetchReply, pm); } else { - /* need more data, do we have space? */ - if (size >= 4096) - goto completed; - else - storeClientCopy(pm->running.sc, pm->running.e, size, 0, 4096, buf, peerMonitorFetchReplyHeaders, pm); + pm->running.size = 0; + pm->running.offset = reply->hdr_sz; } + storeClientCopy(pm->running.sc, pm->running.e, pm->running.offset, pm->running.offset, 4096, buf, peerMonitorFetchReply, pm); return; completed: Index: squid/src/urn.c =================================================================== RCS file: /cvsroot/squid/squid/src/urn.c,v retrieving revision 1.82 retrieving revision 1.82.2.1 diff -u -r1.82 -r1.82.2.1 --- squid/src/urn.c 26 Feb 2007 09:11:10 -0000 1.82 +++ squid/src/urn.c 8 Apr 2008 22:52:59 -0000 1.82.2.1 @@ -1,6 +1,6 @@ /* - * $Id: urn.c,v 1.82 2007/02/26 09:11:10 hno Exp $ + * $Id: urn.c,v 1.82.2.1 2008/04/08 22:52:59 hno Exp $ * * DEBUG: section 52 URN Parsing * AUTHOR: Kostas Anagnostakis @@ -177,7 +177,6 @@ StoreEntry *e = urnState->entry; StoreEntry *urlres_e = urnState->urlres_e; char *s = NULL; - size_t k; HttpReply *rep; url_entry *urls; url_entry *u; @@ -209,16 +208,6 @@ urnState); return; } - /* we know its STORE_OK */ - k = headersEnd(buf, size); - if (0 == k) { - debug(52, 1) ("urnHandleReply: didn't find end-of-headers for %s\n", - storeUrl(e)); - return; - } - s = buf + k; - assert(urlres_e->mem_obj->reply); - httpReplyParse(urlres_e->mem_obj->reply, buf, k); debug(52, 3) ("mem->reply exists, code=%d.\n", urlres_e->mem_obj->reply->sline.status); if (urlres_e->mem_obj->reply->sline.status != HTTP_OK) { @@ -228,13 +217,25 @@ errorAppendEntry(e, err); return; } - while (xisspace(*s)) + /* Make sure the data is null terminated so we can parse it as a string */ + if (size == SM_PAGE_SIZE) + size--; + s = buf + urlres_e->mem_obj->reply->hdr_sz; + size -= urlres_e->mem_obj->reply->hdr_sz; + if (size < 0) + goto error; + while (xisspace(*s) && size > 0) { s++; + size--; + } + s = xstrndup(s, size); urls = urnParseReply(s, urnState->request->method); + safe_free(s); for (i = 0; NULL != urls[i].url; i++) urlcnt++; debug(52, 3) ("urnHandleReply: Counted %d URLs\n", i); if (urls == NULL) { /* unkown URN error */ + error: debug(52, 3) ("urnHandleReply: unknown URN %s\n", storeUrl(e)); err = errorCon(ERR_URN_RESOLVE, HTTP_NOT_FOUND, urnState->request); err->url = xstrdup(storeUrl(e));