--------------------- PatchSet 11800 Date: 2007/12/14 20:05:24 Author: hno Branch: HEAD Tag: (none) Log: Bug #1893: Variant invalidation on PURGE and HTCP CLR This patch contributed by Wikipedia adds PURGE and HTCP CLR support on Vary:ing objects by adding a kind of horizon to the Vary details, making Squid reject cached variants from a different horizon. Members: src/client_side.c:1.753->1.754 src/comm_generic.c:1.9->1.10 src/enums.h:1.244->1.245 src/http.c:1.438->1.439 src/protos.h:1.546->1.547 src/store.c:1.583->1.584 src/store_client.c:1.126->1.127 src/store_swapmeta.c:1.22->1.23 src/structs.h:1.537->1.538 src/typedefs.h:1.156->1.157 Index: squid/src/client_side.c =================================================================== RCS file: /cvsroot/squid/squid/src/client_side.c,v retrieving revision 1.753 retrieving revision 1.754 diff -u -r1.753 -r1.754 --- squid/src/client_side.c 13 Dec 2007 01:20:49 -0000 1.753 +++ squid/src/client_side.c 14 Dec 2007 20:05:24 -0000 1.754 @@ -1,6 +1,6 @@ /* - * $Id: client_side.c,v 1.753 2007/12/13 01:20:49 hno Exp $ + * $Id: client_side.c,v 1.754 2007/12/14 20:05:24 hno Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -2289,6 +2289,18 @@ case VARY_CANCEL: /* varyEvaluateMatch found a object loop. Process as miss */ debug(33, 1) ("clientProcessHit: Vary object loop!\n"); + storeClientUnregister(http->sc, e, http); + http->sc = NULL; + clientProcessMiss(http); + return; + case VARY_EXPIRED: + /* Variant is expired. Delete it and process as a miss. */ + debug(33, 2) ("clientProcessHit: Variant expired, deleting\n"); + storeClientUnregister(http->sc, e, http); + http->sc = NULL; + storeRelease(e); + storeUnlockObject(e); + http->entry = NULL; clientProcessMiss(http); return; } @@ -4993,6 +5005,8 @@ */ vary = httpMakeVaryMark(request, entry->mem_obj->reply); if (vary) { + /* Save the vary_id for the second time through. */ + request->vary_id = entry->mem_obj->vary_id; return VARY_OTHER; } else { /* Ouch.. we cannot handle this kind of variance */ @@ -5010,6 +5024,13 @@ /* This request was merged before we knew the outcome. Don't trust the response */ /* restart vary processing from the beginning */ return VARY_RESTART; + } else if (request->vary_id.create_time != entry->mem_obj->vary_id.create_time || + request->vary_id.serial != entry->mem_obj->vary_id.serial) { + /* vary_id mismatch, the variant must be expired */ + debug(33, 3) ("varyEvaluateMatch: vary ID mismatch, parent is %ld.%u, child is %ld.%u\n", + request->vary_id.create_time, request->vary_id.serial, + entry->mem_obj->vary_id.create_time, entry->mem_obj->vary_id.serial); + return VARY_EXPIRED; } else { return VARY_MATCH; } Index: squid/src/comm_generic.c =================================================================== RCS file: /cvsroot/squid/squid/src/comm_generic.c,v retrieving revision 1.9 retrieving revision 1.10 diff -u -r1.9 -r1.10 --- squid/src/comm_generic.c 13 Aug 2007 03:10:53 -0000 1.9 +++ squid/src/comm_generic.c 14 Dec 2007 20:05:24 -0000 1.10 @@ -1,6 +1,6 @@ /* - * $Id: comm_generic.c,v 1.9 2007/08/13 03:10:53 hno Exp $ + * $Id: comm_generic.c,v 1.10 2007/12/14 20:05:24 hno Exp $ * * DEBUG: section 5 Socket Functions * @@ -360,7 +360,7 @@ int rc; double start = current_dtime; - debug(5, 3) ("comm_select: timeout %d\n", msec); + debug(5, 5) ("comm_select: timeout %d\n", msec); if (msec > MAX_POLL_TIME) msec = MAX_POLL_TIME; Index: squid/src/enums.h =================================================================== RCS file: /cvsroot/squid/squid/src/enums.h,v retrieving revision 1.244 retrieving revision 1.245 diff -u -r1.244 -r1.245 --- squid/src/enums.h 13 Dec 2007 01:25:34 -0000 1.244 +++ squid/src/enums.h 14 Dec 2007 20:05:24 -0000 1.245 @@ -1,6 +1,6 @@ /* - * $Id: enums.h,v 1.244 2007/12/13 01:25:34 hno Exp $ + * $Id: enums.h,v 1.245 2007/12/14 20:05:24 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -660,6 +660,7 @@ STORE_META_STD_LFS, /* standard metadata in lfs format */ STORE_META_OBJSIZE, /* object size, if its known */ STORE_META_STOREURL, /* the store url, if different to the normal URL */ + STORE_META_VARY_ID, /* Unique ID linking variants */ STORE_META_END }; @@ -753,7 +754,8 @@ VARY_MATCH, VARY_OTHER, VARY_RESTART, - VARY_CANCEL + VARY_CANCEL, + VARY_EXPIRED }; /* Windows Port */ Index: squid/src/http.c =================================================================== RCS file: /cvsroot/squid/squid/src/http.c,v retrieving revision 1.438 retrieving revision 1.439 diff -u -r1.438 -r1.439 --- squid/src/http.c 13 Dec 2007 01:20:49 -0000 1.438 +++ squid/src/http.c 14 Dec 2007 20:05:24 -0000 1.439 @@ -1,6 +1,6 @@ /* - * $Id: http.c,v 1.438 2007/12/13 01:20:49 hno Exp $ + * $Id: http.c,v 1.439 2007/12/14 20:05:24 hno Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -229,7 +229,7 @@ const char *v; #if HTTP_VIOLATIONS const refresh_t *R = NULL; - /* This strange looking define first looks up the frefresh pattern + /* This strange looking define first looks up the refresh pattern * and then checks if the specified flag is set. The main purpose * of this is to simplify the refresh pattern lookup */ Index: squid/src/protos.h =================================================================== RCS file: /cvsroot/squid/squid/src/protos.h,v retrieving revision 1.546 retrieving revision 1.547 diff -u -r1.546 -r1.547 --- squid/src/protos.h 13 Dec 2007 01:20:49 -0000 1.546 +++ squid/src/protos.h 14 Dec 2007 20:05:24 -0000 1.547 @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.546 2007/12/13 01:20:49 hno Exp $ + * $Id: protos.h,v 1.547 2007/12/14 20:05:24 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1457,7 +1457,7 @@ /* ETag support */ void storeLocateVaryDone(VaryData * data); void storeLocateVary(StoreEntry * e, int offset, const char *vary_data, String accept_encoding, STLVCB * callback, void *cbdata); -void storeAddVary(const char *url, const method_t method, const cache_key * key, const char *etag, const char *vary, const char *vary_headers, const char *accept_encoding); +vary_id_t storeAddVary(const char *url, const method_t method, const cache_key * key, const char *etag, const char *vary, const char *vary_headers, const char *accept_encoding); /* New HTTP message parsing support */ extern void HttpMsgBufInit(HttpMsgBuf * hmsg, const char *buf, size_t size); Index: squid/src/store.c =================================================================== RCS file: /cvsroot/squid/squid/src/store.c,v retrieving revision 1.583 retrieving revision 1.584 diff -u -r1.583 -r1.584 --- squid/src/store.c 13 Dec 2007 01:25:35 -0000 1.583 +++ squid/src/store.c 14 Dec 2007 20:05:24 -0000 1.584 @@ -1,6 +1,6 @@ /* - * $Id: store.c,v 1.583 2007/12/13 01:25:35 hno Exp $ + * $Id: store.c,v 1.584 2007/12/14 20:05:24 hno Exp $ * * DEBUG: section 20 Storage Manager * AUTHOR: Harvest Derived @@ -87,6 +87,7 @@ static void storeEntryDereferenced(StoreEntry *); static int getKeyCounter(void); static int storeKeepInMemory(const StoreEntry *); +static void initVaryId(vary_id_t *); static OBJH storeCheckCachableStats; static EVH storeLateRelease; @@ -347,8 +348,9 @@ StoreEntry * storeGet(const cache_key * key) { - debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key)); - return (StoreEntry *) hash_lookup(store_table, key); + StoreEntry *e = (StoreEntry *) hash_lookup(store_table, key); + debug(20, 3) ("storeGet: %s -> %p\n", storeKeyText(key), e); + return e; } StoreEntry * @@ -452,6 +454,11 @@ storeAppendPrintf(state->e, "ETag: %s\n", state->etag); storeAppendPrintf(state->e, "VaryData: %s\n", state->vary_headers); } + if (state->oe) { + debug(11, 3) ("free_AddVaryState: copying vary ID %ld.%u to new entry\n", + state->oe->mem_obj->vary_id.create_time, state->oe->mem_obj->vary_id.serial); + state->e->mem_obj->vary_id = state->oe->mem_obj->vary_id; + } storeTimestampsSet(state->e); storeComplete(state->e); storeTimestampsSet(state->e); @@ -563,6 +570,7 @@ debug(11, 3) ("storeAddVaryReadOld: %p seen_offset=%" PRINTF_OFF_T " buf_offset=%d size=%d\n", data, state->seen_offset, (int) state->buf_offset, (int) size); if (size <= 0) { debug(11, 2) ("storeAddVaryReadOld: DONE\n"); + /* Call back to the destructor free_AddVaryState */ cbdataFree(state); return; } @@ -706,12 +714,14 @@ /* * Adds/updates a Vary record. - * For updates only one of key or etag needs to be specified - * At leas one of key or etag must be specified, preferably both. + * At least one of key or etag must be specified, preferably both. + * Returns the vary ID if it can be determined immediately, zero otherwise */ -void +vary_id_t storeAddVary(const char *url, const method_t method, const cache_key * key, const char *etag, const char *vary, const char *vary_headers, const char *accept_encoding) { + vary_id_t vary_id = + {0, 0}; AddVaryState *state; request_flags flags = null_request_flags; CBDATA_INIT_TYPE_FREECB(AddVaryState, free_AddVaryState); @@ -733,6 +743,11 @@ state->e = storeCreateEntry(url, flags, method); httpReplySetHeaders(state->e->mem_obj->reply, HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000); httpHeaderPutStr(&state->e->mem_obj->reply->header, HDR_VARY, vary); + if (!state->oe) { + /* New entry, create new unique ID */ + initVaryId(&vary_id); + state->e->mem_obj->vary_id = vary_id; + } storeSetPublicKey(state->e); storeBuffer(state->e); httpReplySwapOut(state->e->mem_obj->reply, state->e); @@ -751,8 +766,10 @@ * - VaryData is added last in the Key record it corresponds to (after * modifications above) */ - /* Swap in the dummy Vary object */ - if (!state->oe->mem_obj) { + if (state->oe->mem_obj) { + vary_id = state->oe->mem_obj->vary_id; + } else { + /* Swap in the dummy Vary object. vary_id is unknown for now */ storeCreateMemObject(state->oe, state->url); state->oe->mem_obj->method = method; } @@ -764,10 +781,10 @@ state->buf, storeAddVaryReadOld, state); - return; } else { cbdataFree(state); } + return vary_id; } static MemPool *VaryData_pool = NULL; @@ -810,9 +827,10 @@ static void storeLocateVaryCallback(LocateVaryState * state) { + int expired = FALSE; if (cbdataValid(state->callback_data)) { VaryData *data = state->data; - if (data->key || data->etags.count) { + if (!expired && (data->key || data->etags.count)) { state->callback(data, state->callback_data); state->data = NULL; /* now owned by the caller */ } else { @@ -989,6 +1007,7 @@ StoreEntry *e2 = NULL; const cache_key *newkey; MemObject *mem = e->mem_obj; + debug(20, 3) ("storeSetPublicKey: %s\n", storeKeyText(e->hash.key)); if (e->hash.key && !EBIT_TEST(e->flags, KEY_PRIVATE)) { if (EBIT_TEST(e->flags, KEY_EARLY_PUBLIC)) { EBIT_CLR(e->flags, KEY_EARLY_PUBLIC); @@ -1042,6 +1061,7 @@ newkey = storeKeyPublicByRequest(mem->request); if (mem->vary_headers && !EBIT_TEST(e->flags, KEY_EARLY_PUBLIC)) { String vary = StringNull; + vary_id_t vary_id; String varyhdr; varyhdr = httpHeaderGetList(&mem->reply->header, HDR_VARY); if (strBuf(varyhdr)) @@ -1054,7 +1074,26 @@ strListAdd(&vary, strBuf(varyhdr), ','); stringClean(&varyhdr); #endif - storeAddVary(mem->url, mem->method, newkey, httpHeaderGetStr(&mem->reply->header, HDR_ETAG), strBuf(vary), mem->vary_headers, mem->vary_encoding); + /* Create or update the vary object */ + vary_id = storeAddVary(mem->url, mem->method, newkey, httpHeaderGetStr(&mem->reply->header, HDR_ETAG), strBuf(vary), mem->vary_headers, mem->vary_encoding); + if (vary_id.create_time) { + mem->vary_id = vary_id; + } else { + /* Base vary object is not swapped in, so the vary_id is unknown. + * Maybe we can cheat and use the vary_id from the request. If the + * base object existed earlier in the request, it would have been + * swapped in and stored at that time. + */ + if (mem->request->vary_id.create_time) { + mem->vary_id = mem->request->vary_id; + } else { + /* Nope, no luck. Store with zero vary_id, which will immediately + * be treated as expired. + * FIXME: make this work properly. + */ + debug(20, 1) ("storeSetPublicKey: unable to determine vary_id for '%s'\n", mem->url); + } + } stringClean(&vary); } } else { @@ -1799,6 +1838,8 @@ debug(20, 1) ("MemObject->url: %p %s\n", mem->url, checkNullString(mem->url)); + debug(20, 1) ("MemObject->vary_id: %ld.%u\n", + mem->vary_id.create_time, mem->vary_id.serial); } void @@ -2082,3 +2123,15 @@ if (e->mem_obj) e->mem_obj->serverfd = -1; } + +/* Initialise the vary_id with a new unique value */ +static void +initVaryId(vary_id_t * vary_id) +{ + static unsigned int serial = 0; + + debug(20, 3) ("initVaryId: Initialising vary_id to %ld.%u\n", + squid_curtime, serial); + vary_id->create_time = squid_curtime; + vary_id->serial = serial++; +} Index: squid/src/store_client.c =================================================================== RCS file: /cvsroot/squid/squid/src/store_client.c,v retrieving revision 1.126 retrieving revision 1.127 diff -u -r1.126 -r1.127 --- squid/src/store_client.c 29 Nov 2007 12:18:42 -0000 1.126 +++ squid/src/store_client.c 14 Dec 2007 20:05:24 -0000 1.127 @@ -1,6 +1,6 @@ /* - * $Id: store_client.c,v 1.126 2007/11/29 12:18:42 adrian Exp $ + * $Id: store_client.c,v 1.127 2007/12/14 20:05:24 hno Exp $ * * DEBUG: section 20 Storage Manager Client-Side Interface * AUTHOR: Duane Wessels @@ -440,6 +440,9 @@ mem->vary_headers = xstrdup(t->value); } break; + case STORE_META_VARY_ID: + memcpy(&mem->vary_id, t->value, sizeof(vary_id_t)); + break; default: debug(20, 2) ("WARNING: got unused STORE_META type %d\n", t->type); break; Index: squid/src/store_swapmeta.c =================================================================== RCS file: /cvsroot/squid/squid/src/store_swapmeta.c,v retrieving revision 1.22 retrieving revision 1.23 diff -u -r1.22 -r1.23 --- squid/src/store_swapmeta.c 16 Nov 2007 11:38:46 -0000 1.22 +++ squid/src/store_swapmeta.c 14 Dec 2007 20:05:24 -0000 1.23 @@ -1,6 +1,6 @@ /* - * $Id: store_swapmeta.c,v 1.22 2007/11/16 11:38:46 adrian Exp $ + * $Id: store_swapmeta.c,v 1.23 2007/12/14 20:05:24 hno Exp $ * * DEBUG: section 20 Storage Manager Swapfile Metadata * AUTHOR: Kostas Anagnostakis @@ -88,6 +88,7 @@ T = storeSwapTLVAdd(STORE_META_VARY_HEADERS, vary, strlen(vary) + 1, T); if (e->mem_obj->store_url) T = storeSwapTLVAdd(STORE_META_STOREURL, e->mem_obj->store_url, strlen(e->mem_obj->store_url) + 1, T); + storeSwapTLVAdd(STORE_META_VARY_ID, &e->mem_obj->vary_id, sizeof(vary_id_t), T); return TLV; } Index: squid/src/structs.h =================================================================== RCS file: /cvsroot/squid/squid/src/structs.h,v retrieving revision 1.537 retrieving revision 1.538 diff -u -r1.537 -r1.538 --- squid/src/structs.h 13 Dec 2007 01:25:35 -0000 1.537 +++ squid/src/structs.h 14 Dec 2007 20:05:24 -0000 1.538 @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.537 2007/12/13 01:25:35 hno Exp $ + * $Id: structs.h,v 1.538 2007/12/14 20:05:24 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1715,6 +1715,11 @@ void (*Done) (RemovalPurgeWalker * walker); }; +struct _vary_id_t { + time_t create_time; + unsigned int serial; +}; + /* This structure can be freed while object is purged out from memory */ struct _MemObject { method_t method; @@ -1755,6 +1760,7 @@ StoreEntry *old_entry; time_t refresh_timestamp; time_t stale_while_revalidate; + vary_id_t vary_id; }; struct _StoreEntry { @@ -1937,6 +1943,7 @@ char *vary_headers; /* Used when varying entities are detected. Changes how the store key is calculated */ String vary_encoding; /* Used when varying entities are detected. Changes how the store key is calculated. */ VaryData *vary; + vary_id_t vary_id; /* Vary ID of the parent vary object */ Array *etags; /* possible known entity tags (Vary MISS) */ char *etag; /* current entity tag, cache validation */ unsigned int done_etag:1; /* We have done clientProcessETag on this, don't attempt it again */ Index: squid/src/typedefs.h =================================================================== RCS file: /cvsroot/squid/squid/src/typedefs.h,v retrieving revision 1.156 retrieving revision 1.157 diff -u -r1.156 -r1.157 --- squid/src/typedefs.h 23 Nov 2007 11:06:47 -0000 1.156 +++ squid/src/typedefs.h 14 Dec 2007 20:05:24 -0000 1.157 @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.156 2007/11/23 11:06:47 hno Exp $ + * $Id: typedefs.h,v 1.157 2007/12/14 20:05:24 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -226,6 +226,7 @@ typedef struct _RemovalPolicySettings RemovalPolicySettings; typedef struct _errormap errormap; typedef struct _PeerMonitor PeerMonitor; +typedef struct _vary_id_t vary_id_t; typedef struct _http_version_t http_version_t;