---------------------
PatchSet 11758 
Date: 2007/11/23 11:06:46
Author: hno
Branch: HEAD
Tag: (none) 
Log:
external_refresh_check helper, allowing program control over cache refreshes

this helper is called whenever Squid needs to refresh a cached entry,
and allows full control on if the refresh really needs to be performed
to the backend server or if the cached copy is still fresh.

If fresh new header details may also be returned by the helper, updating
the cached headers.

Members: 
	src/Makefile.am:1.65->1.66 
	src/access_log.c:1.103->1.104 
	src/cache_cf.c:1.477->1.478 
	src/cf.data.depend:1.1->1.2 
	src/cf.data.pre:1.443->1.444 
	src/client_side.c:1.747->1.748 
	src/main.c:1.402->1.403 
	src/protos.h:1.544->1.545 
	src/refresh_check.c:INITIAL->1.1 
	src/structs.h:1.533->1.534 
	src/typedefs.h:1.155->1.156 

Index: squid/src/Makefile.am
===================================================================
RCS file: /cvsroot/squid/squid/src/Makefile.am,v
retrieving revision 1.65
retrieving revision 1.66
diff -u -r1.65 -r1.66
--- squid/src/Makefile.am	21 Nov 2007 13:44:29 -0000	1.65
+++ squid/src/Makefile.am	23 Nov 2007 11:06:46 -0000	1.66
@@ -1,7 +1,7 @@
 #
 #  Makefile for the Squid Object Cache server
 #
-#  $Id: Makefile.am,v 1.65 2007/11/21 13:44:29 hno Exp $
+#  $Id: Makefile.am,v 1.66 2007/11/23 11:06:46 hno Exp $
 #
 #  Uncomment and customize the following to suit your needs:
 #
@@ -228,6 +228,7 @@
 	store_rewrite.c \
 	referer.c \
 	refresh.c \
+	refresh_check.c \
 	send-announce.c \
 	$(SNMPSOURCE) \
 	squid.h \
Index: squid/src/access_log.c
===================================================================
RCS file: /cvsroot/squid/squid/src/access_log.c,v
retrieving revision 1.103
retrieving revision 1.104
diff -u -r1.103 -r1.104
--- squid/src/access_log.c	23 Sep 2007 14:48:55 -0000	1.103
+++ squid/src/access_log.c	23 Nov 2007 11:06:46 -0000	1.104
@@ -1,6 +1,6 @@
 
 /*
- * $Id: access_log.c,v 1.103 2007/09/23 14:48:55 adrian Exp $
+ * $Id: access_log.c,v 1.104 2007/11/23 11:06:46 hno Exp $
  *
  * DEBUG: section 46    Access Log
  * AUTHOR: Duane Wessels
@@ -320,6 +320,8 @@
 
     LFT_SEQUENCE_NUMBER,
 
+    LFT_EXT_FRESHNESS,
+
     LFT_PERCENT			/* special string cases for escaped chars */
 } logformat_bcode_t;
 
@@ -428,6 +430,8 @@
 
     {"ea", LFT_EXT_LOG},
 
+    {"ef", LFT_EXT_FRESHNESS},
+
     {"%", LFT_PERCENT},
 
     {NULL, LFT_NONE}		/* this must be last */
@@ -691,6 +695,12 @@
 	    doint = 1;
 	    break;
 
+	case LFT_EXT_FRESHNESS:
+	    out = al->ext_refresh;
+
+	    quote = 1;
+	    break;
+
 	case LFT_PERCENT:
 	    out = "%";
 	    break;
Index: squid/src/cache_cf.c
===================================================================
RCS file: /cvsroot/squid/squid/src/cache_cf.c,v
retrieving revision 1.477
retrieving revision 1.478
diff -u -r1.477 -r1.478
--- squid/src/cache_cf.c	21 Nov 2007 15:06:13 -0000	1.477
+++ squid/src/cache_cf.c	23 Nov 2007 11:06:46 -0000	1.478
@@ -1,6 +1,6 @@
 
 /*
- * $Id: cache_cf.c,v 1.477 2007/11/21 15:06:13 hno Exp $
+ * $Id: cache_cf.c,v 1.478 2007/11/23 11:06:46 hno Exp $
  *
  * DEBUG: section 3     Configuration File Parsing
  * AUTHOR: Harvest Derived
@@ -479,6 +479,7 @@
     requirePathnameExists("Error Directory", Config.errorDirectory);
     authenticateConfigure(&Config.authConfig);
     externalAclConfigure();
+    refreshCheckConfigure();
 #if HTTP_VIOLATIONS
     {
 	const refresh_t *R;
Index: squid/src/cf.data.depend
===================================================================
RCS file: /cvsroot/squid/squid/src/cf.data.depend,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- squid/src/cf.data.depend	5 Sep 2007 21:25:21 -0000	1.1
+++ squid/src/cf.data.depend	23 Nov 2007 11:06:46 -0000	1.2
@@ -52,3 +52,4 @@
 programline
 extension_method
 errormap
+refreshCheckHelper
Index: squid/src/cf.data.pre
===================================================================
RCS file: /cvsroot/squid/squid/src/cf.data.pre,v
retrieving revision 1.443
retrieving revision 1.444
diff -u -r1.443 -r1.444
--- squid/src/cf.data.pre	23 Nov 2007 06:59:26 -0000	1.443
+++ squid/src/cf.data.pre	23 Nov 2007 11:06:46 -0000	1.444
@@ -1,6 +1,6 @@
 
 #
-# $Id: cf.data.pre,v 1.443 2007/11/23 06:59:26 hno Exp $
+# $Id: cf.data.pre,v 1.444 2007/11/23 11:06:46 hno Exp $
 #
 # SQUID Web Proxy Cache          http://www.squid-cache.org/
 # ----------------------------------------------------------
@@ -3335,6 +3335,69 @@
 	or response to be rejected.
 DOC_END
 
+NAME: external_refresh_check
+TYPE: refreshCheckHelper
+DEFAULT: none
+LOC: Config.Program.refresh_check
+DOC_START
+	This option defines an external helper for determining whether to
+	refresh a stale response. It will be called when Squid receives a
+	request for a cached response that is stale; the helper can either
+	confirm that the response is stale with a STALE response, or
+	extend the freshness of the response (thereby avoiding a refresh
+	check) with a FRESH response, along with a freshness=nnn keyword.
+
+	  external_refresh_check [options] FORMAT.. /path/to/helper [helper_args]
+
+	If present, helper_args will be passed to the helper on the command
+	line verbatim.
+
+	Options:
+
+	  children=n	Number of processes to spawn to service external
+			refresh checks (default 5).
+	  concurrency=n	Concurrency level per process. Only used with
+			helpers capable of processing more than one query
+			at a time.
+
+	When using the concurrency option, the protocol is changed by introducing
+	a query channel tag infront of the request/response. The query channel
+	tag is a number between 0 and concurrency−1.
+
+	FORMAT specifications:
+
+	  %CACHE_URI	The URI of the cached response
+	  %RES{Header}	HTTP response header value
+	  %AGE		The age of the cached response
+
+	The request sent to the helper consists of the data in the format
+	specification in the order specified.
+
+	The helper receives lines per the above format specification, and
+	returns lines starting with OK or ERR indicating the validity of
+	the request and optionally followed by additional keywords with
+	more details.  URL escaping is used to protect each value in both
+	requests and responses.
+
+	General result syntax:
+
+	  FRESH / STALE keyword=value ...
+
+	Defined keywords:
+
+	  freshness=nnn	The number of seconds to extend the freshness of
+			the response by.
+	  log=string	String to be logged in access.log. Available as
+			%ef in logformat specifications.
+	  res{Header}=value
+			Value to update response headers with. If already
+			present, the supplied value completely replaces
+			the cached value.
+
+	In the event of a helper−related error (e.g., overload), Squid
+	will always default to STALE.
+DOC_END
+
 COMMENT_START
  TIMEOUTS
  -----------------------------------------------------------------------------
Index: squid/src/client_side.c
===================================================================
RCS file: /cvsroot/squid/squid/src/client_side.c,v
retrieving revision 1.747
retrieving revision 1.748
diff -u -r1.747 -r1.748
--- squid/src/client_side.c	21 Nov 2007 15:06:13 -0000	1.747
+++ squid/src/client_side.c	23 Nov 2007 11:06:46 -0000	1.748
@@ -1,6 +1,6 @@
 
 /*
- * $Id: client_side.c,v 1.747 2007/11/21 15:06:13 hno Exp $
+ * $Id: client_side.c,v 1.748 2007/11/23 11:06:46 hno Exp $
  *
  * DEBUG: section 33    Client-side Routines
  * AUTHOR: Duane Wessels
@@ -116,6 +116,7 @@
 static int clientGotNotEnough(clientHttpRequest *);
 static void checkFailureRatio(err_type, hier_code);
 static void clientProcessMiss(clientHttpRequest *);
+static void clientProcessHit(clientHttpRequest * http);
 static void clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep);
 static clientHttpRequest *parseHttpRequestAbort(ConnStateData * conn, const char *uri);
 static clientHttpRequest *parseHttpRequest(ConnStateData *, HttpMsgBuf *, method_t *, int *);
@@ -141,6 +142,8 @@
 static void clientPackTermBound(String boundary, MemBuf * mb);
 static void clientProcessRequest(clientHttpRequest *);
 static void clientProcessExpired(clientHttpRequest *);
+static void clientRefreshCheck(clientHttpRequest *);
+static REFRESHCHECK clientRefreshCheckDone;
 static void clientProcessOnlyIfCachedMiss(clientHttpRequest * http);
 static int clientCachable(clientHttpRequest * http);
 static int clientHierarchical(clientHttpRequest * http);
@@ -667,6 +670,26 @@
 }
 
 static void
+clientRefreshCheck(clientHttpRequest * http)
+{
+    refreshCheckSubmit(http->entry, clientRefreshCheckDone, http);
+}
+
+static void
+clientRefreshCheckDone(void *data, int fresh, const char *log)
+{
+    clientHttpRequest *http = data;
+    if (log) {
+	safe_free(http->al.ext_refresh);
+	http->al.ext_refresh = xstrdup(log);
+    }
+    if (fresh)
+	clientProcessHit(http);
+    else
+	clientProcessExpired(http);
+}
+
+static void
 clientProcessExpired(clientHttpRequest * http)
 {
     char *url = http->uri;
@@ -2322,6 +2345,7 @@
 	http->log_type = LOG_TCP_STALE_HIT;
 	stale = 0;
     }
+    http->is_modified = is_modified;
     if (stale) {
 	debug(33, 5) ("clientCacheHit: in refreshCheck() block\n");
 	/*
@@ -2344,9 +2368,18 @@
 	    clientProcessMiss(http);
 	    return;
 	}
-	clientProcessExpired(http);
+	clientRefreshCheck(http);
 	return;
     }
+    clientProcessHit(http);
+}
+
+static void
+clientProcessHit(clientHttpRequest * http)
+{
+    int is_modified = http->is_modified;
+    StoreEntry *e = http->entry;
+
     if (is_modified == 0) {
 	time_t timestamp = e->timestamp;
 	MemBuf mb = httpPacked304Reply(e->mem_obj->reply);
@@ -2377,7 +2410,7 @@
 	http->log_type = LOG_TCP_MISS;
     else if (http->log_type == LOG_TCP_HIT && e->mem_status == IN_MEMORY)
 	http->log_type = LOG_TCP_MEM_HIT;
-    clientSendHeaders(data, rep);
+    clientSendHeaders(http, e->mem_obj->reply);
 }
 
 /* put terminating boundary for multiparts */
Index: squid/src/main.c
===================================================================
RCS file: /cvsroot/squid/squid/src/main.c,v
retrieving revision 1.402
retrieving revision 1.403
diff -u -r1.402 -r1.403
--- squid/src/main.c	15 Nov 2007 04:43:55 -0000	1.402
+++ squid/src/main.c	23 Nov 2007 11:06:47 -0000	1.403
@@ -1,6 +1,6 @@
 
 /*
- * $Id: main.c,v 1.402 2007/11/15 04:43:55 adrian Exp $
+ * $Id: main.c,v 1.403 2007/11/23 11:06:47 hno Exp $
  *
  * DEBUG: section 1     Startup and Main Loop
  * AUTHOR: Harvest Derived
@@ -400,6 +400,7 @@
     locationRewriteShutdown();
     authenticateShutdown();
     externalAclShutdown();
+    refreshCheckShutdown();
     storeDirCloseSwapLogs();
     storeLogClose();
     accessLogClose();
@@ -430,6 +431,7 @@
     locationRewriteInit();
     authenticateInit(&Config.authConfig);
     externalAclInit();
+    refreshCheckInit();
 #if USE_WCCP
     wccpInit();
 #endif
@@ -465,6 +467,7 @@
     locationRewriteShutdown();
     authenticateShutdown();
     externalAclShutdown();
+    refreshCheckShutdown();
     _db_rotate_log();		/* cache.log */
     storeDirWriteCleanLogs(1);
     storeDirSync();		/* Flush pending I/O ops */
@@ -484,6 +487,7 @@
     locationRewriteInit();
     authenticateInit(&Config.authConfig);
     externalAclInit();
+    refreshCheckInit();
 }
 
 static void
@@ -584,6 +588,7 @@
     errorMapInit();
     authenticateInit(&Config.authConfig);
     externalAclInit();
+    refreshCheckInit();
     useragentOpenLog();
     refererOpenLog();
     httpHeaderInitModule();	/* must go before any header processing (e.g. the one in errorInitialize) */
@@ -1093,6 +1098,7 @@
 #endif
     redirectShutdown();
     externalAclShutdown();
+    refreshCheckShutdown();
     locationRewriteShutdown();
     icpConnectionClose();
 #if USE_HTCP
Index: squid/src/protos.h
===================================================================
RCS file: /cvsroot/squid/squid/src/protos.h,v
retrieving revision 1.544
retrieving revision 1.545
diff -u -r1.544 -r1.545
--- squid/src/protos.h	21 Nov 2007 15:06:13 -0000	1.544
+++ squid/src/protos.h	23 Nov 2007 11:06:47 -0000	1.545
@@ -1,6 +1,6 @@
 
 /*
- * $Id: protos.h,v 1.544 2007/11/21 15:06:13 hno Exp $
+ * $Id: protos.h,v 1.545 2007/11/23 11:06:47 hno Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -1417,6 +1417,15 @@
 const char *externalAclMessage(external_acl_entry * entry);
 
 
+/* refresh_check.c */
+extern void parse_refreshCheckHelper(refresh_check_helper **);
+extern void dump_refreshCheckHelper(StoreEntry * sentry, const char *name, const refresh_check_helper *);
+extern void free_refreshCheckHelper(refresh_check_helper **);
+extern void refreshCheckSubmit(StoreEntry * entry, REFRESHCHECK * callback, void *data);
+extern void refreshCheckInit(void);
+extern void refreshCheckConfigure(void);
+extern void refreshCheckShutdown(void);
+
 #if USE_WCCPv2
 extern void parse_wccp2_service(void *v);
 extern void free_wccp2_service(void *v);
--- /dev/null	Fri Nov 23 11:53:32 2007
+++ squid/src/refresh_check.c	Fri Nov 23 11:53:32 2007
@@ -0,0 +1,549 @@
+
+/*
+ * $Id: refresh_check.c,v 1.1 2007/11/23 11:06:47 hno Exp $
+ *
+ * DEBUG: section 82    External ACL
+ * AUTHOR: Henrik Nordstrom
+ *
+ * SQUID Web Proxy Cache          http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ *  The contents of this file is Copyright (C) 2002 by MARA Systems AB,
+ *  Sweden, unless otherwise is indicated in the specific function. The
+ *  author gives his full permission to include this file into the Squid
+ *  software product under the terms of the GNU General Public License as
+ *  published by the Free Software Foundation; either version 2 of the
+ *  License, or (at your option) any later version.
+ *
+ *  Squid is the result of efforts by numerous individuals from
+ *  the Internet community; see the CONTRIBUTORS file for full
+ *  details.   Many organizations have provided support for Squid's
+ *  development; see the SPONSORS file for full details.  Squid is
+ *  Copyrighted (C) 2001 by the Regents of the University of
+ *  California; see the COPYRIGHT file for full details.  Squid
+ *  incorporates software developed and/or copyrighted by other
+ *  sources; see the CREDITS file for full details.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+
+#ifndef DEFAULT_REFRESH_CHECK_CHILDREN
+#define DEFAULT_REFRESH_CHECK_CHILDREN 5
+#endif
+
+/******************************************************************
+ * external_refresh_check parser
+ */
+typedef struct _refresh_check_format refresh_check_format;
+
+struct _refresh_check_helper {
+    refresh_check_format *format;
+    wordlist *cmdline;
+    int children;
+    int concurrency;
+    helper *helper;
+    dlink_list queue;
+};
+
+struct _refresh_check_format {
+    enum {
+	REFRESH_CHECK_UNKNOWN,
+	REFRESH_CHECK_URI,
+	REFRESH_CHECK_RESP_HEADER,
+	REFRESH_CHECK_RESP_HEADER_MEMBER,
+	REFRESH_CHECK_RESP_HEADER_ID,
+	REFRESH_CHECK_RESP_HEADER_ID_MEMBER,
+	REFRESH_CHECK_AGE,
+	REFRESH_CHECK_END
+    } type;
+    refresh_check_format *next;
+    char *header;
+    char *member;
+    char separator;
+    http_hdr_type header_id;
+};
+
+/* FIXME: These are not really cbdata, but it is an easy way
+ * to get them pooled, refcounted, accounted and freed properly...
+ */
+CBDATA_TYPE(refresh_check_helper);
+CBDATA_TYPE(refresh_check_format);
+
+static void
+free_refresh_check_format(void *data)
+{
+    refresh_check_format *p = data;
+    safe_free(p->header);
+}
+
+static void
+free_refresh_check_helper(void *data)
+{
+    refresh_check_helper *p = data;
+    while (p->format) {
+	refresh_check_format *f = p->format;
+	p->format = f->next;
+	cbdataFree(f);
+    }
+    wordlistDestroy(&p->cmdline);
+    if (p->helper) {
+	helperShutdown(p->helper);
+	helperFree(p->helper);
+	p->helper = NULL;
+    }
+}
+
+void
+parse_refreshCheckHelper(refresh_check_helper ** ptr)
+{
+    refresh_check_helper *a;
+    char *token;
+    refresh_check_format **p;
+
+    if (*ptr)
+	self_destruct();
+
+    CBDATA_INIT_TYPE_FREECB(refresh_check_helper, free_refresh_check_helper);
+    CBDATA_INIT_TYPE_FREECB(refresh_check_format, free_refresh_check_format);
+
+    a = cbdataAlloc(refresh_check_helper);
+
+    a->children = DEFAULT_REFRESH_CHECK_CHILDREN;
+
+    /* Parse options */
+    while ((token = strtok(NULL, w_space)) != NULL) {
+	if (strncmp(token, "children=", 9) == 0) {
+	    a->children = atoi(token + 9);
+	} else if (strncmp(token, "concurrency=", 12) == 0) {
+	    a->concurrency = atoi(token + 12);
+	} else {
+	    break;
+	}
+    }
+
+    /* Parse format */
+    p = &a->format;
+    while (token) {
+	refresh_check_format *format;
+
+	/* stop on first non-format token found */
+	if (*token != '%')
+	    break;
+
+	format = cbdataAlloc(refresh_check_format);
+
+	if (strncmp(token, "%RES{", 5) == 0) {
+	    /* header format */
+	    char *header, *member, *end;
+	    header = token + 5;
+	    end = strchr(header, '}');
+	    /* cut away the terminating } */
+	    if (end && strlen(end) == 1)
+		*end = '\0';
+	    else
+		self_destruct();
+
+	    member = strchr(header, ':');
+	    if (member) {
+		/* Split in header and member */
+		*member++ = '\0';
+		if (!xisalnum(*member))
+		    format->separator = *member++;
+		else
+		    format->separator = ',';
+		format->member = xstrdup(member);
+		format->type = REFRESH_CHECK_RESP_HEADER_MEMBER;
+	    } else {
+		format->type = REFRESH_CHECK_RESP_HEADER;
+	    }
+	    format->header = xstrdup(header);
+	    format->header_id = httpHeaderIdByNameDef(header, strlen(header));
+	    if (format->header_id != -1) {
+		if (member)
+		    format->type = REFRESH_CHECK_RESP_HEADER_ID_MEMBER;
+		else
+		    format->type = REFRESH_CHECK_RESP_HEADER_ID;
+	    }
+	} else if (strcmp(token, "%URI") == 0)
+	    format->type = REFRESH_CHECK_URI;
+	else if (strcmp(token, "%URL") == 0)
+	    format->type = REFRESH_CHECK_URI;
+	else if (strcmp(token, "%CACHE_URI") == 0)
+	    format->type = REFRESH_CHECK_URI;
+	else if (strcmp(token, "%AGE") == 0)
+	    format->type = REFRESH_CHECK_AGE;
+	else {
+	    self_destruct();
+	}
+	*p = format;
+	p = &format->next;
+	token = strtok(NULL, w_space);
+    }
+    /* There must be at least one format token */
+    if (!a->format)
+	self_destruct();
+
+    /* helper */
+    if (!token)
+	self_destruct();
+    wordlistAdd(&a->cmdline, token);
+
+    /* arguments */
+    parse_wordlist(&a->cmdline);
+
+    *ptr = a;
+}
+
+void
+dump_refreshCheckHelper(StoreEntry * sentry, const char *name, const refresh_check_helper * list)
+{
+    const refresh_check_helper *node = list;
+    const refresh_check_format *format;
+    const wordlist *word;
+    if (node) {
+	storeAppendPrintf(sentry, "%s", name);
+	if (node->children != DEFAULT_REFRESH_CHECK_CHILDREN)
+	    storeAppendPrintf(sentry, " children=%d", node->children);
+	if (node->concurrency)
+	    storeAppendPrintf(sentry, " concurrency=%d", node->concurrency);
+	for (format = node->format; format; format = format->next) {
+	    switch (format->type) {
+	    case REFRESH_CHECK_RESP_HEADER:
+	    case REFRESH_CHECK_RESP_HEADER_ID:
+		storeAppendPrintf(sentry, " %%{%s}", format->header);
+		break;
+	    case REFRESH_CHECK_RESP_HEADER_MEMBER:
+	    case REFRESH_CHECK_RESP_HEADER_ID_MEMBER:
+		storeAppendPrintf(sentry, " %%{%s:%s}", format->header, format->member);
+		break;
+#define DUMP_REFRESH_CHECK_TYPE(a) \
+	    case REFRESH_CHECK_##a: \
+		storeAppendPrintf(sentry, " %%%s", #a); \
+		break
+		DUMP_REFRESH_CHECK_TYPE(URI);
+		DUMP_REFRESH_CHECK_TYPE(AGE);
+	    case REFRESH_CHECK_UNKNOWN:
+	    case REFRESH_CHECK_END:
+		fatal("unknown refresh_check format error");
+		break;
+	    }
+	}
+	for (word = node->cmdline; word; word = word->next)
+	    storeAppendPrintf(sentry, " %s", word->key);
+	storeAppendPrintf(sentry, "\n");
+    }
+}
+
+void
+free_refreshCheckHelper(refresh_check_helper ** list)
+{
+    while (*list) {
+	refresh_check_helper *node = *list;
+	*list = NULL;
+	cbdataFree(node);
+    }
+}
+
+/******************************************************************
+ * refresh_check runtime
+ */
+
+static inline int
+refreshCheckOverload(refresh_check_helper * def)
+{
+    return def->helper->stats.queue_size > def->helper->n_running;
+}
+
+static char *
+makeRefreshCheckRequest(StoreEntry * entry, refresh_check_format * format)
+{
+    static MemBuf mb = MemBufNULL;
+    int first = 1;
+    HttpReply *reply;
+    String sb = StringNull;
+
+    if (!entry->mem_obj)
+	return NULL;
+
+    reply = entry->mem_obj->reply;
+    memBufReset(&mb);
+    for (; format; format = format->next) {
+	char buf[256];
+	const char *str = NULL;
+	const char *quoted;
+	switch (format->type) {
+	case REFRESH_CHECK_URI:
+	    str = entry->mem_obj->url;
+	    break;
+	case REFRESH_CHECK_AGE:
+	    snprintf(buf, sizeof(buf), "%ld", (long int) (squid_curtime - entry->timestamp));
+	    str = buf;
+	    break;
+	case REFRESH_CHECK_RESP_HEADER:
+	    sb = httpHeaderGetByName(&reply->header, format->header);
+	    str = strBuf(sb);
+	    break;
+	case REFRESH_CHECK_RESP_HEADER_ID:
+	    sb = httpHeaderGetStrOrList(&reply->header, format->header_id);
+	    str = strBuf(sb);
+	    break;
+	case REFRESH_CHECK_RESP_HEADER_MEMBER:
+	    sb = httpHeaderGetByNameListMember(&reply->header, format->header, format->member, format->separator);
+	    str = strBuf(sb);
+	    break;
+	case REFRESH_CHECK_RESP_HEADER_ID_MEMBER:
+	    sb = httpHeaderGetListMember(&reply->header, format->header_id, format->member, format->separator);
+	    str = strBuf(sb);
+	    break;
+
+	case REFRESH_CHECK_UNKNOWN:
+	case REFRESH_CHECK_END:
+	    fatal("unknown refresh_check_program format error");
+	    break;
+	}
+	if (!str || !*str)
+	    str = "-";
+	if (!first)
+	    memBufAppend(&mb, " ", 1);
+	quoted = rfc1738_escape(str);
+	memBufAppend(&mb, quoted, strlen(quoted));
+	stringClean(&sb);
+	first = 0;
+    }
+    return mb.buf;
+}
+
+/******************************************************************
+ * refresh_check requests
+ */
+
+typedef struct _refreshCheckState refreshCheckState;
+struct _refreshCheckState {
+    REFRESHCHECK *callback;
+    void *callback_data;
+    StoreEntry *entry;
+    refresh_check_helper *def;
+    dlink_node list;
+    refreshCheckState *queue;
+};
+
+CBDATA_TYPE(refreshCheckState);
+static void
+free_refreshCheckState(void *data)
+{
+    refreshCheckState *state = data;
+    storeUnlockObject(state->entry);
+    cbdataUnlock(state->callback_data);
+    cbdataUnlock(state->def);
+}
+
+/*
+ * The helper program receives queries on stdin, one
+ * per line, and must return the result on on stdout
+ */
+static void
+refreshCheckHandleReply(void *data, char *reply)
+{
+    refreshCheckState *state = data;
+    refreshCheckState *next;
+    int freshness = -1;
+    char *log = NULL;
+    MemBuf hdrs = MemBufNULL;
+
+
+    debug(82, 2) ("refreshCheckHandleReply: reply=\"%s\"\n", reply);
+
+    if (reply) {
+	char *t = NULL;
+	char *token = strwordtok(reply, &t);
+	if (token && strcmp(token, "FRESH") == 0)
+	    freshness = 0;
+	else if (token && strcmp(token, "OK") == 0)
+	    freshness = 0;
+
+	while ((token = strwordtok(NULL, &t))) {
+	    char *value = strchr(token, '=');
+	    if (value) {
+		*value++ = '\0';	/* terminate the token, and move up to the value */
+		rfc1738_unescape(value);
+		if (strcmp(token, "freshness") == 0)
+		    freshness = atoi(value);
+		else if (strcmp(token, "log") == 0)
+		    log = value;
+		else if (strncmp(token, "res{", 4) == 0) {
+		    char *header, *t;
+		    header = token + 4;
+		    t = strrchr(header, '}');
+		    if (!t)
+			continue;
+		    *t = '\0';
+		    if (!hdrs.buf)
+			memBufDefInit(&hdrs);
+		    memBufPrintf(&hdrs, "%s: %s\r\n", header, value);
+		}
+	    }
+	}
+    }
+    if (freshness >= 0) {
+	if (hdrs.size) {
+	    HttpReply *rep = httpReplyCreate();
+	    httpHeaderParse(&rep->header, hdrs.buf, hdrs.buf + hdrs.size);
+	    httpReplyUpdateOnNotModified(state->entry->mem_obj->reply, rep);
+	    storeTimestampsSet(state->entry);
+	    if (!httpHeaderHas(&rep->header, HDR_DATE)) {
+		state->entry->timestamp = squid_curtime;
+		state->entry->expires = squid_curtime + freshness;
+	    } else if (freshness) {
+		state->entry->expires = squid_curtime + freshness;
+	    }
+	    httpReplyDestroy(rep);
+	    storeUpdate(state->entry, NULL);
+	} else {
+	    state->entry->timestamp = squid_curtime;
+	    state->entry->expires = squid_curtime + freshness;
+	}
+    }
+    if (hdrs.buf)
+	memBufClean(&hdrs);
+    dlinkDelete(&state->list, &state->def->queue);
+    do {
+	cbdataUnlock(state->def);
+	state->def = NULL;
+
+	if (state->callback && cbdataValid(state->callback_data))
+	    state->callback(state->callback_data, freshness >= 0, log);
+	cbdataUnlock(state->callback_data);
+	state->callback_data = NULL;
+
+	next = state->queue;
+	cbdataFree(state);
+	state = next;
+    } while (state);
+}
+
+void
+refreshCheckSubmit(StoreEntry * entry, REFRESHCHECK * callback, void *callback_data)
+{
+    MemBuf buf;
+    const char *key;
+    refresh_check_helper *def = Config.Program.refresh_check;
+    refreshCheckState *state;
+    dlink_node *node;
+    refreshCheckState *oldstate = NULL;
+
+    if (!def) {
+	callback(callback_data, 0, NULL);
+	return;
+    }
+    key = makeRefreshCheckRequest(entry, def->format);
+    if (!key) {
+	callback(callback_data, 0, NULL);
+	return;
+    }
+    debug(82, 2) ("refreshCheckLookup: for '%s'\n", key);
+
+    /* Check for a pending lookup to hook into */
+    for (node = def->queue.head; node; node = node->next) {
+	refreshCheckState *oldstatetmp = node->data;
+	if (entry == oldstatetmp->entry) {
+	    oldstate = oldstatetmp;
+	    break;
+	}
+    }
+
+    state = cbdataAlloc(refreshCheckState);
+    state->def = def;
+    cbdataLock(state->def);
+    state->entry = entry;
+    storeLockObject(entry);
+    state->callback = callback;
+    state->callback_data = callback_data;
+    cbdataLock(state->callback_data);
+    if (oldstate) {
+	/* Hook into pending lookup */
+	state->queue = oldstate->queue;
+	oldstate->queue = state;
+    } else {
+	/* No pending lookup found. Sumbit to helper */
+	/* Check for queue overload */
+	if (refreshCheckOverload(def)) {
+	    debug(82, 1) ("refreshCheckLookup: queue overload\n");
+	    cbdataFree(state);
+	    callback(callback_data, 0, "Overload");
+	    return;
+	}
+	/* Send it off to the helper */
+	memBufDefInit(&buf);
+	memBufPrintf(&buf, "%s\n", key);
+	helperSubmit(def->helper, buf.buf, refreshCheckHandleReply, state);
+	dlinkAdd(state, &state->list, &def->queue);
+	memBufClean(&buf);
+    }
+}
+
+static void
+refreshCheckStats(StoreEntry * sentry)
+{
+    refresh_check_helper *p = Config.Program.refresh_check;
+
+    if (p) {
+	storeAppendPrintf(sentry, "Refresh Check helper statistics:\n");
+	helperStats(sentry, p->helper);
+	storeAppendPrintf(sentry, "\n");
+    }
+}
+void
+refreshCheckConfigure(void)
+{
+    refresh_check_helper *p = Config.Program.refresh_check;
+    if (p) {
+	requirePathnameExists("external_refresh_check", p->cmdline->key);
+    }
+}
+void
+refreshCheckInit(void)
+{
+    static int firstTimeInit = 1;
+    refresh_check_helper *p = Config.Program.refresh_check;
+
+    if (p) {
+	if (!p->helper)
+	    p->helper = helperCreate("external_refresh_check");
+	p->helper->cmdline = p->cmdline;
+	p->helper->n_to_start = p->children;
+	p->helper->concurrency = p->concurrency;
+	p->helper->ipc_type = IPC_STREAM;
+	helperOpenServers(p->helper);
+	if (firstTimeInit) {
+	    firstTimeInit = 0;
+	    cachemgrRegister("refresh_check",
+		"External ACL stats",
+		refreshCheckStats, 0, 1);
+	}
+	CBDATA_INIT_TYPE_FREECB(refreshCheckState, free_refreshCheckState);
+    }
+}
+
+void
+refreshCheckShutdown(void)
+{
+    refresh_check_helper *p = Config.Program.refresh_check;
+    if (p) {
+	helperShutdown(p->helper);
+    }
+}
Index: squid/src/structs.h
===================================================================
RCS file: /cvsroot/squid/squid/src/structs.h,v
retrieving revision 1.533
retrieving revision 1.534
diff -u -r1.533 -r1.534
--- squid/src/structs.h	23 Nov 2007 06:59:26 -0000	1.533
+++ squid/src/structs.h	23 Nov 2007 11:06:47 -0000	1.534
@@ -1,6 +1,6 @@
 
 /*
- * $Id: structs.h,v 1.533 2007/11/23 06:59:26 hno Exp $
+ * $Id: structs.h,v 1.534 2007/11/23 11:06:47 hno Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -556,6 +556,7 @@
 	    int children;
 	    int concurrency;
 	} location_rewrite;
+	refresh_check_helper *refresh_check;
 #if USE_ICMP
 	char *pinger;
 #endif
@@ -1199,6 +1200,7 @@
     HierarchyLogEntry hier;
     HttpReply *reply;
     request_t *request;
+    char *ext_refresh;
 };
 
 struct _clientHttpRequest {
@@ -1248,6 +1250,7 @@
     squid_off_t maxBodySize;
     STHCB *header_callback;	/* Temporarily here for storeClientCopyHeaders */
     StoreEntry *header_entry;	/* Temporarily here for storeClientCopyHeaders */
+    int is_modified;
 };
 
 struct _ConnStateData {
Index: squid/src/typedefs.h
===================================================================
RCS file: /cvsroot/squid/squid/src/typedefs.h,v
retrieving revision 1.155
retrieving revision 1.156
diff -u -r1.155 -r1.156
--- squid/src/typedefs.h	21 Nov 2007 15:06:13 -0000	1.155
+++ squid/src/typedefs.h	23 Nov 2007 11:06:47 -0000	1.156
@@ -1,6 +1,6 @@
 
 /*
- * $Id: typedefs.h,v 1.155 2007/11/21 15:06:13 hno Exp $
+ * $Id: typedefs.h,v 1.156 2007/11/23 11:06:47 hno Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -419,4 +419,6 @@
 typedef void LOGROTATE(Logfile *);
 typedef void LOGCLOSE(Logfile *);
 
+typedef void REFRESHCHECK(void *data, int fresh, const char *log);
+typedef struct _refresh_check_helper refresh_check_helper;
 #endif /* SQUID_TYPEDEFS_H */