------------------------------------------------------------ revno: 12272 revision-id: kinkie@squid-cache.org-20120810065649-asftqg1oyypm20z1 parent: squid3@treenet.co.nz-20120810055658-ptfx61ks4sv36kvr committer: Francesco Chemolli branch nick: trunk timestamp: Fri 2012-08-10 08:56:49 +0200 message: SourceLayout: introduce Fs::Ufs namespace, split ufscommon, store_dir_ufs and store_io_ufs ------------------------------------------------------------ # Bazaar merge directive format 2 (Bazaar 0.90) # revision_id: kinkie@squid-cache.org-20120810065649-asftqg1oyypm20z1 # target_branch: http://bzr.squid-cache.org/bzr/squid3/trunk/ # testament_sha1: 27a49358a369a62f0a82e79ca51ef4342ba07ca7 # timestamp: 2012-08-10 07:26:53 +0000 # source_branch: http://bzr.squid-cache.org/bzr/squid3/trunk/ # base_revision_id: squid3@treenet.co.nz-20120810055658-\ # ptfx61ks4sv36kvr # # Begin patch === modified file 'src/fs/Makefile.am' --- src/fs/Makefile.am 2011-09-10 16:38:36 +0000 +++ src/fs/Makefile.am 2012-08-10 06:56:49 +0000 @@ -22,11 +22,18 @@ libufs_la_SOURCES = \ ufs/StoreFSufs.h \ ufs/StoreFSufs.cc \ - ufs/store_dir_ufs.cc \ - ufs/store_io_ufs.cc \ - ufs/ufscommon.cci \ - ufs/ufscommon.cc \ - ufs/ufscommon.h + ufs/UFSStoreState.cc \ + ufs/UFSSwapDir.cc \ + ufs/UFSSwapDir.h \ + ufs/UFSStrategy.cc \ + ufs/UFSStrategy.h \ + ufs/UFSStoreState.h \ + ufs/StoreSearchUFS.h \ + ufs/StoreSearchUFS.cc \ + ufs/UFSSwapLogParser.h \ + ufs/UFSSwapLogParser.cc \ + ufs/RebuildState.h \ + ufs/RebuildState.cc librock_la_SOURCES = \ rock/RockDbCell.h \ === modified file 'src/fs/Module.cc' --- src/fs/Module.cc 2012-01-20 18:55:04 +0000 +++ src/fs/Module.cc 2012-08-10 06:56:49 +0000 @@ -2,7 +2,7 @@ #include "Module.h" #if defined(HAVE_FS_UFS) || defined(HAVE_FS_AUFS) || defined(HAVE_FS_DISKD) #include "fs/ufs/StoreFSufs.h" -#include "fs/ufs/ufscommon.h" +#include "fs/ufs/UFSSwapDir.h" #endif #if HAVE_FS_COSS @@ -10,16 +10,16 @@ #endif #if HAVE_FS_UFS -static StoreFSufs *UfsInstance; +static Fs::Ufs::StoreFSufs *UfsInstance; #endif #if HAVE_FS_AUFS -static StoreFSufs *AufsInstance; +static Fs::Ufs::StoreFSufs *AufsInstance; #endif #if HAVE_FS_DISKD -static StoreFSufs *DiskdInstance; +static Fs::Ufs::StoreFSufs *DiskdInstance; #endif #if HAVE_FS_ROCK @@ -42,16 +42,16 @@ { #if HAVE_FS_UFS - UfsInstance = new StoreFSufs("Blocking", "ufs"); + UfsInstance = new Fs::Ufs::StoreFSufs("Blocking", "ufs"); #endif #if HAVE_FS_AUFS - AufsInstance = new StoreFSufs("DiskThreads", "aufs");; + AufsInstance = new Fs::Ufs::StoreFSufs("DiskThreads", "aufs");; #endif #if HAVE_FS_DISKD - DiskdInstance = new StoreFSufs("DiskDaemon", "diskd");; + DiskdInstance = new Fs::Ufs::StoreFSufs("DiskDaemon", "diskd");; #endif #if HAVE_FS_ROCK === modified file 'src/fs/aufs/StoreFSaufs.cc' --- src/fs/aufs/StoreFSaufs.cc 2012-01-20 18:55:04 +0000 +++ src/fs/aufs/StoreFSaufs.cc 2012-08-10 06:56:49 +0000 @@ -45,8 +45,7 @@ #include "fs/ufs/StoreFSufs.h" -/** \todo FIXME: break UFSSwapDir out so we don't need all the guff */ -#include "fs/ufs/ufscommon.h" +#include "fs/ufs/UFSSwapDir.h" /** \defgroup AUFS AUFS Storage Filesystem (UFS Based) @@ -54,4 +53,3 @@ */ /* Unused variable: */ -StoreFSufs *AufsInstance_foo = NULL; === modified file 'src/fs/diskd/StoreFSdiskd.cc' --- src/fs/diskd/StoreFSdiskd.cc 2012-01-20 18:55:04 +0000 +++ src/fs/diskd/StoreFSdiskd.cc 2012-08-10 06:56:49 +0000 @@ -42,9 +42,7 @@ #endif #include "fs/ufs/StoreFSufs.h" - -/** \todo FIXME: break UFSSwapDir out so we don;t need all the extras */ -#include "fs/ufs/ufscommon.h" +#include "fs/ufs/UFSSwapDir.h" /** \defgroup diskd diskd Storage Filesystem (UFS Based) @@ -52,4 +50,4 @@ */ /* Unused variable: */ -StoreFSufs *DiskdInstance_foo = NULL; +Fs::Ufs::StoreFSufs *DiskdInstance_foo = NULL; === modified file 'src/fs/rock/RockSwapDir.cc' --- src/fs/rock/RockSwapDir.cc 2012-06-22 03:49:38 +0000 +++ src/fs/rock/RockSwapDir.cc 2012-08-10 06:56:49 +0000 @@ -18,6 +18,7 @@ #include "MemObject.h" #include "Parsing.h" #include "SquidMath.h" +#include #include const int64_t Rock::SwapDir::HeaderSize = 16*1024; === added file 'src/fs/ufs/RebuildState.cc' --- src/fs/ufs/RebuildState.cc 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/RebuildState.cc 2012-08-10 06:56:49 +0000 @@ -0,0 +1,540 @@ +/* + * + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "RebuildState.h" +#include "SquidTime.h" +#include "StoreSwapLogData.h" +#include "UFSSwapLogParser.h" + +CBDATA_NAMESPACED_CLASS_INIT(Fs::Ufs,RebuildState); + +Fs::Ufs::RebuildState::RebuildState(RefCount aSwapDir) : + sd (aSwapDir), LogParser(NULL), e(NULL), fromLog(true), _done (false) +{ + /* + * If the swap.state file exists in the cache_dir, then + * we'll use commonUfsDirRebuildFromSwapLog(), otherwise we'll + * use commonUfsDirRebuildFromDirectory() to open up each file + * and suck in the meta data. + */ + int clean = 0; + int zeroLengthLog = 0; + FILE *fp = sd->openTmpSwapLog(&clean, &zeroLengthLog); + + if (fp && !zeroLengthLog) + LogParser = Fs::Ufs::UFSSwapLogParser::GetUFSSwapLogParser(fp); + + if (LogParser == NULL ) { + fromLog = false; + + if (fp != NULL) + fclose(fp); + + } else { + fromLog = true; + flags.clean = (unsigned int) clean; + } + + if (!clean) + flags.need_to_validate = 1; + + debugs(47, DBG_IMPORTANT, "Rebuilding storage in " << sd->path << " (" << + (clean ? "clean log" : (LogParser ? "dirty log" : "no log")) << ")"); +} + +Fs::Ufs::RebuildState::~RebuildState() +{ + sd->closeTmpSwapLog(); + + if (LogParser) + delete LogParser; +} + +void +Fs::Ufs::RebuildState::RebuildStep(void *data) +{ + RebuildState *rb = (RebuildState *)data; + rb->rebuildStep(); + + if (!rb->isDone()) + eventAdd("storeRebuild", RebuildStep, rb, 0.01, 1); + else { + -- StoreController::store_dirs_rebuilding; + storeRebuildComplete(&rb->counts); + delete rb; + } +} + +/// load entries from swap.state or files until we run out of entries or time +void +Fs::Ufs::RebuildState::rebuildStep() +{ + currentEntry(NULL); + + // Balance our desire to maximize the number of entries processed at once + // (and, hence, minimize overheads and total rebuild time) with a + // requirement to also process Coordinator events, disk I/Os, etc. + const int maxSpentMsec = 50; // keep small: most RAM I/Os are under 1ms + const timeval loopStart = current_time; + + const int totalEntries = LogParser ? LogParser->SwapLogEntries() : -1; + + while (!isDone()) { + if (fromLog) + rebuildFromSwapLog(); + else + rebuildFromDirectory(); + + // TODO: teach storeRebuildProgress to handle totalEntries <= 0 + if (totalEntries > 0 && (n_read % 4000 == 0)) + storeRebuildProgress(sd->index, totalEntries, n_read); + + if (opt_foreground_rebuild) + continue; // skip "few entries at a time" check below + + getCurrentTime(); + const double elapsedMsec = tvSubMsec(loopStart, current_time); + if (elapsedMsec > maxSpentMsec || elapsedMsec < 0) { + debugs(47, 5, HERE << "pausing after " << n_read << " entries in " << + elapsedMsec << "ms; " << (elapsedMsec/n_read) << "ms per entry"); + break; + } + } +} + +/// process one cache file +void +Fs::Ufs::RebuildState::rebuildFromDirectory() +{ + cache_key key[SQUID_MD5_DIGEST_LENGTH]; + + struct stat sb; + int fd = -1; + assert(this != NULL); + debugs(47, 3, HERE << "DIR #" << sd->index); + + assert(fd == -1); + sfileno filn = 0; + int size; + fd = getNextFile(&filn, &size); + + if (fd == -2) { + debugs(47, DBG_IMPORTANT, "Done scanning " << sd->path << " dir (" << + n_read << " entries)"); + _done = true; + return; + } else if (fd < 0) { + return; + } + + assert(fd > -1); + /* lets get file stats here */ + + ++n_read; + + if (fstat(fd, &sb) < 0) { + debugs(47, DBG_IMPORTANT, HERE << "fstat(FD " << fd << "): " << xstrerror()); + file_close(fd); + --store_open_disk_fd; + fd = -1; + return; + } + + MemBuf buf; + buf.init(SM_PAGE_SIZE, SM_PAGE_SIZE); + if (!storeRebuildLoadEntry(fd, sd->index, buf, counts)) + return; + + StoreEntry tmpe; + const bool loaded = storeRebuildParseEntry(buf, tmpe, key, counts, + (int64_t)sb.st_size); + + file_close(fd); + --store_open_disk_fd; + fd = -1; + + if (!loaded) { + // XXX: shouldn't this be a call to commonUfsUnlink? + sd->unlinkFile(filn); // should we unlink in all failure cases? + return; + } + + if (!storeRebuildKeepEntry(tmpe, key, counts)) + return; + + ++counts.objcount; + // tmpe.dump(5); + currentEntry(sd->addDiskRestore(key, + filn, + tmpe.swap_file_sz, + tmpe.expires, + tmpe.timestamp, + tmpe.lastref, + tmpe.lastmod, + tmpe.refcount, /* refcount */ + tmpe.flags, /* flags */ + (int) flags.clean)); + storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); +} + +StoreEntry * +Fs::Ufs::RebuildState::currentEntry() const +{ + return e; +} + +void +Fs::Ufs::RebuildState::currentEntry(StoreEntry *newValue) +{ + e = newValue; +} + +/// process one swap log entry +void +Fs::Ufs::RebuildState::rebuildFromSwapLog() +{ + StoreSwapLogData swapData; + + if (LogParser->ReadRecord(swapData) != 1) { + debugs(47, DBG_IMPORTANT, "Done reading " << sd->path << " swaplog (" << n_read << " entries)"); + LogParser->Close(); + delete LogParser; + LogParser = NULL; + _done = true; + return; + } + + ++n_read; + + if (!swapData.sane()) { + ++counts.invalid; + return; + } + + /* + * BC: during 2.4 development, we changed the way swap file + * numbers are assigned and stored. The high 16 bits used + * to encode the SD index number. There used to be a call + * to storeDirProperFileno here that re-assigned the index + * bits. Now, for backwards compatibility, we just need + * to mask it off. + */ + swapData.swap_filen &= 0x00FFFFFF; + + debugs(47, 3, HERE << swap_log_op_str[(int) swapData.op] << " " << + storeKeyText(swapData.key) << " "<< std::setfill('0') << + std::hex << std::uppercase << std::setw(8) << + swapData.swap_filen); + + if (swapData.op == SWAP_LOG_ADD) { + (void) 0; + } else if (swapData.op == SWAP_LOG_DEL) { + /* Delete unless we already have a newer copy anywhere in any store */ + /* this needs to become + * 1) unpack url + * 2) make synthetic request with headers ?? or otherwise search + * for a matching object in the store + * TODO FIXME change to new async api + */ + currentEntry (Store::Root().get(swapData.key)); + + if (currentEntry() != NULL && swapData.lastref >= e->lastref) { + undoAdd(); + --counts.objcount; + ++counts.cancelcount; + } + return; + } else { + const double + x = ::log(static_cast(++counts.bad_log_op)) / ::log(10.0); + + if (0.0 == x - (double) (int) x) + debugs(47, DBG_IMPORTANT, "WARNING: " << counts.bad_log_op << " invalid swap log entries found"); + + ++counts.invalid; + + return; + } + + ++counts.scancount; // XXX: should not this be incremented earlier? + + if (!sd->validFileno(swapData.swap_filen, 0)) { + ++counts.invalid; + return; + } + + if (EBIT_TEST(swapData.flags, KEY_PRIVATE)) { + ++counts.badflags; + return; + } + + /* this needs to become + * 1) unpack url + * 2) make synthetic request with headers ?? or otherwise search + * for a matching object in the store + * TODO FIXME change to new async api + */ + currentEntry (Store::Root().get(swapData.key)); + + int used; /* is swapfile already in use? */ + + used = sd->mapBitTest(swapData.swap_filen); + + /* If this URL already exists in the cache, does the swap log + * appear to have a newer entry? Compare 'lastref' from the + * swap log to e->lastref. */ + /* is the log entry newer than current entry? */ + int disk_entry_newer = currentEntry() ? (swapData.lastref > currentEntry()->lastref ? 1 : 0) : 0; + + if (used && !disk_entry_newer) { + /* log entry is old, ignore it */ + ++counts.clashcount; + return; + } else if (used && currentEntry() && currentEntry()->swap_filen == swapData.swap_filen && currentEntry()->swap_dirn == sd->index) { + /* swapfile taken, same URL, newer, update meta */ + + if (currentEntry()->store_status == STORE_OK) { + currentEntry()->lastref = swapData.timestamp; + currentEntry()->timestamp = swapData.timestamp; + currentEntry()->expires = swapData.expires; + currentEntry()->lastmod = swapData.lastmod; + currentEntry()->flags = swapData.flags; + currentEntry()->refcount += swapData.refcount; + sd->dereference(*currentEntry()); + } else { + debug_trap("commonUfsDirRebuildFromSwapLog: bad condition"); + debugs(47, DBG_IMPORTANT, HERE << "bad condition"); + } + return; + } else if (used) { + /* swapfile in use, not by this URL, log entry is newer */ + /* This is sorta bad: the log entry should NOT be newer at this + * point. If the log is dirty, the filesize check should have + * caught this. If the log is clean, there should never be a + * newer entry. */ + debugs(47, DBG_IMPORTANT, "WARNING: newer swaplog entry for dirno " << + sd->index << ", fileno "<< std::setfill('0') << std::hex << + std::uppercase << std::setw(8) << swapData.swap_filen); + + /* I'm tempted to remove the swapfile here just to be safe, + * but there is a bad race condition in the NOVM version if + * the swapfile has recently been opened for writing, but + * not yet opened for reading. Because we can't map + * swapfiles back to StoreEntrys, we don't know the state + * of the entry using that file. */ + /* We'll assume the existing entry is valid, probably because + * were in a slow rebuild and the the swap file number got taken + * and the validation procedure hasn't run. */ + assert(flags.need_to_validate); + ++counts.clashcount; + return; + } else if (currentEntry() && !disk_entry_newer) { + /* key already exists, current entry is newer */ + /* keep old, ignore new */ + ++counts.dupcount; + return; + } else if (currentEntry()) { + /* key already exists, this swapfile not being used */ + /* junk old, load new */ + undoAdd(); + --counts.objcount; + ++counts.dupcount; + } else { + /* URL doesnt exist, swapfile not in use */ + /* load new */ + (void) 0; + } + + ++counts.objcount; + + currentEntry(sd->addDiskRestore(swapData.key, + swapData.swap_filen, + swapData.swap_file_sz, + swapData.expires, + swapData.timestamp, + swapData.lastref, + swapData.lastmod, + swapData.refcount, + swapData.flags, + (int) flags.clean)); + + storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); +} + +/// undo the effects of adding an entry in rebuildFromSwapLog() +void +Fs::Ufs::RebuildState::undoAdd() +{ + StoreEntry *added = currentEntry(); + assert(added); + currentEntry(NULL); + + // TODO: Why bother with these two if we are going to release?! + added->expireNow(); + added->releaseRequest(); + + if (added->swap_filen > -1) { + UFSSwapDir *sde = dynamic_cast(INDEXSD(added->swap_dirn)); + assert(sde); + sde->undoAddDiskRestore(added); + } + + added->release(); +} + +int +Fs::Ufs::RebuildState::getNextFile(sfileno * filn_p, int *size) +{ + int fd = -1; + int dirs_opened = 0; + debugs(47, 3, HERE << "flag=" << flags.init << ", " << + sd->index << ": /"<< std::setfill('0') << std::hex << + std::uppercase << std::setw(2) << curlvl1 << "/" << std::setw(2) << + curlvl2); + + if (done) + return -2; + + while (fd < 0 && done == 0) { + fd = -1; + + if (0 == flags.init) { /* initialize, open first file */ + done = 0; + curlvl1 = 0; + curlvl2 = 0; + in_dir = 0; + flags.init = 1; + assert(Config.cacheSwap.n_configured > 0); + } + + if (0 == in_dir) { /* we need to read in a new directory */ + snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X", + sd->path, + curlvl1, curlvl2); + + if (dirs_opened) + return -1; + + td = opendir(fullpath); + + ++dirs_opened; + + if (td == NULL) { + debugs(47, DBG_IMPORTANT, HERE << "error in opendir (" << fullpath << "): " << xstrerror()); + } else { + entry = readdir(td); /* skip . and .. */ + entry = readdir(td); + + if (entry == NULL && errno == ENOENT) + debugs(47, DBG_IMPORTANT, HERE << "WARNING: directory does not exist!"); + debugs(47, 3, HERE << "Directory " << fullpath); + } + } + + if (td != NULL && (entry = readdir(td)) != NULL) { + ++in_dir; + + if (sscanf(entry->d_name, "%x", &fn) != 1) { + debugs(47, 3, HERE << "invalid entry " << entry->d_name); + continue; + } + + if (!UFSSwapDir::FilenoBelongsHere(fn, sd->index, curlvl1, curlvl2)) { + debugs(47, 3, HERE << std::setfill('0') << + std::hex << std::uppercase << std::setw(8) << fn << + " does not belong in " << std::dec << sd->index << "/" << + curlvl1 << "/" << curlvl2); + + continue; + } + + if (sd->mapBitTest(fn)) { + debugs(47, 3, HERE << "Locked, continuing with next."); + continue; + } + + snprintf(fullfilename, MAXPATHLEN, "%s/%s", + fullpath, entry->d_name); + debugs(47, 3, HERE << "Opening " << fullfilename); + fd = file_open(fullfilename, O_RDONLY | O_BINARY); + + if (fd < 0) + debugs(47, DBG_IMPORTANT, HERE << "error opening " << fullfilename << ": " << xstrerror()); + else + ++store_open_disk_fd; + + continue; + } + + if (td != NULL) + closedir(td); + + td = NULL; + + in_dir = 0; + + if (sd->validL2(++curlvl2)) + continue; + + curlvl2 = 0; + + if (sd->validL1(++curlvl1)) + continue; + + curlvl1 = 0; + + done = 1; + } + + *filn_p = fn; + return fd; +} + +bool +Fs::Ufs::RebuildState::error() const +{ + return false; +} + +bool +Fs::Ufs::RebuildState::isDone() const +{ + return _done; +} + +StoreEntry * +Fs::Ufs::RebuildState::currentItem() +{ + return currentEntry(); +} === added file 'src/fs/ufs/RebuildState.h' --- src/fs/ufs/RebuildState.h 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/RebuildState.h 2012-08-10 06:56:49 +0000 @@ -0,0 +1,102 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_FS_UFS_REBUILDSTATE_H +#define SQUID_FS_UFS_REBUILDSTATE_H + +#include "RefCount.h" +#include "UFSSwapDir.h" +#include "structs.h" +#include "UFSSwapLogParser.h" + +class StoreEntry; + +namespace Fs +{ +namespace Ufs +{ + +/// \ingroup UFS +class RebuildState : public RefCountable +{ +public: + static EVH RebuildStep; + + RebuildState(RefCount sd); + ~RebuildState(); + + virtual bool error() const; + virtual bool isDone() const; + virtual StoreEntry *currentItem(); + + RefCount sd; + int n_read; + /* FILE *log;*/ + Fs::Ufs::UFSSwapLogParser *LogParser; + int curlvl1; + int curlvl2; + + struct { + unsigned int need_to_validate:1; + unsigned int clean:1; + unsigned int init:1; + } flags; + int in_dir; + int done; + int fn; + + dirent_t *entry; + DIR *td; + char fullpath[MAXPATHLEN]; + char fullfilename[MAXPATHLEN]; + + struct _store_rebuild_data counts; + +private: + CBDATA_CLASS2(RebuildState); + void rebuildFromDirectory(); + void rebuildFromSwapLog(); + void rebuildStep(); + void undoAdd(); + int getNextFile(sfileno *, int *size); + StoreEntry *currentEntry() const; + void currentEntry(StoreEntry *); + StoreEntry *e; + bool fromLog; + bool _done; + /// \bug (callback) should be hidden behind a proper human readable name + void (callback)(void *cbdata); + void *cbdata; +}; + + +} /* namespace Ufs */ +} /* namespace Fs */ + +#endif /* SQUID_FS_UFS_REBUILDSTATE_H */ === modified file 'src/fs/ufs/StoreFSufs.cc' --- src/fs/ufs/StoreFSufs.cc 2012-01-20 18:55:04 +0000 +++ src/fs/ufs/StoreFSufs.cc 2012-08-10 06:56:49 +0000 @@ -1,6 +1,4 @@ /* - * $Id$ - * * DEBUG: section 47 Store Directory Routines * AUTHOR: Robert Collins * @@ -41,13 +39,11 @@ #endif #include "fs/ufs/StoreFSufs.h" +#include "fs/ufs/UFSSwapDir.h" #if 0 #include "DiskIO/DiskIOModule.h" #endif -/** \todo FIXME: break UFSSwapDir out so we don't build all the extras */ -#include "fs/ufs/ufscommon.h" - /* Unused variable: */ -StoreFSufs *UfsInstance_foo = NULL; +Fs::Ufs::StoreFSufs *UfsInstance_foo = NULL; === modified file 'src/fs/ufs/StoreFSufs.h' --- src/fs/ufs/StoreFSufs.h 2009-01-21 03:47:47 +0000 +++ src/fs/ufs/StoreFSufs.h 2012-08-10 06:56:49 +0000 @@ -37,10 +37,14 @@ \ingroup FileSystems */ +#include "StoreFileSystem.h" + class DiskIOModule; -#include "StoreFileSystem.h" - +namespace Fs +{ +namespace Ufs +{ /** \ingroup UFS, FileSystems * @@ -52,7 +56,6 @@ template class StoreFSufs : public StoreFileSystem { - public: static StoreFileSystem &GetInstance(); StoreFSufs(char const *DefaultModuleType, char const *label); @@ -108,4 +111,7 @@ initialised = true; } +} /* namespace Ufs */ +} /* namespace Fs */ + #endif /* SQUID_STOREFSUFS_H */ === added file 'src/fs/ufs/StoreSearchUFS.cc' --- src/fs/ufs/StoreSearchUFS.cc 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/StoreSearchUFS.cc 2012-08-10 06:56:49 +0000 @@ -0,0 +1,90 @@ +/* + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "cbdata.h" +#include "StoreSearchUFS.h" +#include "UFSSwapDir.h" + +CBDATA_NAMESPACED_CLASS_INIT(Fs::Ufs,StoreSearchUFS); + +Fs::Ufs::StoreSearchUFS::StoreSearchUFS(RefCount aSwapDir) : + sd(aSwapDir), walker (sd->repl->WalkInit(sd->repl)), + current (NULL), _done (false) +{} + +Fs::Ufs::StoreSearchUFS::~StoreSearchUFS() +{ + walker->Done(walker); + walker = NULL; +} + +void +Fs::Ufs::StoreSearchUFS::next(void (aCallback)(void *cbdata), void *aCallbackArgs) +{ + next(); + aCallback(aCallbackArgs); +} + +bool +Fs::Ufs::StoreSearchUFS::next() +{ + /* the walker API doesn't make sense. the store entries referred to are already readwrite + * from their hash table entries + */ + + if (walker) + current = const_cast(walker->Next(walker)); + + if (current == NULL) + _done = true; + + return current != NULL; +} + +bool +Fs::Ufs::StoreSearchUFS::error() const +{ + return false; +} + +bool +Fs::Ufs::StoreSearchUFS::isDone() const +{ + return _done; +} + +StoreEntry * +Fs::Ufs::StoreSearchUFS::currentItem() +{ + return current; +} === added file 'src/fs/ufs/StoreSearchUFS.h' --- src/fs/ufs/StoreSearchUFS.h 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/StoreSearchUFS.h 2012-08-10 06:56:49 +0000 @@ -0,0 +1,83 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_FS_UFS_STORESEARCHUFS_H +#define SQUID_FS_UFS_STORESEARCHUFS_H + +#include "StoreSearch.h" +#include "UFSSwapDir.h" + +namespace Fs +{ +namespace Ufs +{ + +/// \ingroup UFS +class StoreSearchUFS : public StoreSearch +{ +public: + StoreSearchUFS(RefCount sd); + virtual ~StoreSearchUFS(); + + /** \todo Iterator API - garh, wrong place */ + /** + * callback the client when a new StoreEntry is available + * or an error occurs + */ + virtual void next(void (callback)(void *cbdata), void *cbdata); + + /** + \retval true if a new StoreEntry is immediately available + \retval false if a new StoreEntry is NOT immediately available + */ + virtual bool next(); + + virtual bool error() const; + virtual bool isDone() const; + virtual StoreEntry *currentItem(); + + RefCount sd; + RemovalPolicyWalker *walker; + +private: + CBDATA_CLASS2(StoreSearchUFS); + /// \bug (callback) should be hidden behind a proper human readable name + void (callback)(void *cbdata); + void *cbdata; + StoreEntry * current; + bool _done; + + StoreSearchUFS(StoreSearchUFS const &); //disabled + StoreSearchUFS& operator=(StoreSearchUFS const &); //disabled + StoreSearchUFS(); //disabled +}; + +} //namespace Ufs +} //namespace Fs +#endif /* SQUID_FS_UFS_STORESEARCHUFS_H */ === added file 'src/fs/ufs/UFSStoreState.cc' --- src/fs/ufs/UFSStoreState.cc 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/UFSStoreState.cc 2012-08-10 06:56:49 +0000 @@ -0,0 +1,519 @@ + +/* + * $Id$ + * + * DEBUG: section 79 Storage Manager UFS Interface + * AUTHOR: Duane Wessels + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid-old.h" +#include "Store.h" +#include "Generic.h" +#include "DiskIO/DiskFile.h" +#include "DiskIO/DiskIOStrategy.h" +#include "DiskIO/ReadRequest.h" +#include "DiskIO/WriteRequest.h" + +#include "SwapDir.h" +#include "UFSStrategy.h" +#include "UFSStoreState.h" + + +CBDATA_NAMESPACED_CLASS_INIT(Fs::Ufs,UFSStoreState); + +void * +Fs::Ufs::UFSStoreState::operator new (size_t) +{ + CBDATA_INIT_TYPE(UFSStoreState); + return cbdataAlloc(UFSStoreState); +} + +void +Fs::Ufs::UFSStoreState::operator delete (void *address) +{ + cbdataFree(address); +} + +void +Fs::Ufs::UFSStoreState::ioCompletedNotification() +{ + if (opening) { + opening = false; + debugs(79, 3, "UFSStoreState::ioCompletedNotification: dirno " << + swap_dirn << ", fileno "<< std::setfill('0') << std::hex << + std::setw(8) << swap_filen << " status "<< std::setfill(' ') << + std::dec << theFile->error()); + + assert (FILE_MODE(mode) == O_RDONLY); + openDone(); + + return; + } + + if (creating) { + creating = false; + debugs(79, 3, "UFSStoreState::ioCompletedNotification: dirno " << + swap_dirn << ", fileno "<< std::setfill('0') << std::hex << + std::setw(8) << swap_filen << " status "<< std::setfill(' ') << + std::dec << theFile->error()); + + openDone(); + + return; + } + + assert (!(closing ||opening)); + debugs(79, 3, "diskd::ioCompleted: dirno " << swap_dirn << ", fileno "<< + std::setfill('0') << std::hex << std::setw(8) << swap_filen << + " status "<< std::setfill(' ') << std::dec << theFile->error()); + + /* Ok, notification past open means an error has occured */ + assert (theFile->error()); + tryClosing(); +} + +void +Fs::Ufs::UFSStoreState::openDone() +{ + if (closing) + debugs(0, DBG_CRITICAL, HERE << "already closing in openDone()!?"); + + if (theFile->error()) { + tryClosing(); + return; + } + + if (FILE_MODE(mode) == O_WRONLY) { + drainWriteQueue(); + + } else if ((FILE_MODE(mode) == O_RDONLY) && !closing) { + if (kickReadQueue()) + return; + } + + if (flags.try_closing) + tryClosing(); + + debugs(79, 3, "UFSStoreState::openDone: exiting"); +} + +void +Fs::Ufs::UFSStoreState::closeCompleted() +{ + assert (closing); + debugs(79, 3, "UFSStoreState::closeCompleted: dirno " << swap_dirn << + ", fileno "<< std::setfill('0') << std::hex << std::setw(8) << + swap_filen << " status "<< std::setfill(' ') << std::dec << + theFile->error()); + + if (theFile->error()) { + debugs(79,3,HERE<< "theFile->error() ret " << theFile->error()); + doCloseCallback(DISK_ERROR); + } else { + doCloseCallback(DISK_OK); + } + + closing = false; +} + +/* + * DPW 2006-05-24 + * This close function is called by the higher layer when it has finished + * reading/writing everything, or otherwise wants to close the swap + * file. In the case of writing and using aufs storage, close() might + * be called before any/all data is written, and even before the open + * callback occurs. Thus, we use our tryClosing() method, which knows + * when it is safe to actually signal the lower layer for closing. + */ +void +Fs::Ufs::UFSStoreState::close(int) +{ + debugs(79, 3, "UFSStoreState::close: dirno " << swap_dirn << ", fileno "<< + std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen); + tryClosing(); // UFS does not distinguish different closure types +} + +void +Fs::Ufs::UFSStoreState::read_(char *buf, size_t size, off_t aOffset, STRCB * aCallback, void *aCallbackData) +{ + assert(read.callback == NULL); + assert(read.callback_data == NULL); + assert(!reading); + assert(!closing); + assert (aCallback); + + if (!theFile->canRead()) { + debugs(79, 3, "UFSStoreState::read_: queueing read because theFile can't read"); + queueRead (buf, size, aOffset, aCallback, aCallbackData); + return; + } + + read.callback = aCallback; + read.callback_data = cbdataReference(aCallbackData); + debugs(79, 3, "UFSStoreState::read_: dirno " << swap_dirn << ", fileno "<< + std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen); + offset_ = aOffset; + read_buf = buf; + reading = true; + theFile->read(new ReadRequest(buf,aOffset,size)); +} + + +/* + * DPW 2006-05-24 + * This, the public write interface, places the write request at the end + * of the pending_writes queue to ensure correct ordering of writes. + * We could optimize things a little if there are no other pending + * writes and just do the write directly. But for now we'll keep the + * code simpler and always go through the pending_writes queue. + */ +void +Fs::Ufs::UFSStoreState::write(char const *buf, size_t size, off_t aOffset, FREE * free_func) +{ + debugs(79, 3, "UFSStoreState::write: dirn " << swap_dirn << ", fileno "<< + std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen); + + if (theFile->error()) { + debugs(79, DBG_IMPORTANT,HERE << "avoid write on theFile with error"); + debugs(79, DBG_IMPORTANT,HERE << "calling free_func for " << (void*) buf); + free_func((void*)buf); + return; + } + + queueWrite(buf, size, aOffset, free_func); + drainWriteQueue(); +} + + +/* + * DPW 2006-05-24 + * This, the private write method, calls the lower level write for the + * first write request in the pending_writes queue. doWrite() is only + * called by drainWriteQueue(). + */ +void +Fs::Ufs::UFSStoreState::doWrite() +{ + debugs(79, 3, HERE << this << " UFSStoreState::doWrite"); + + assert(theFile->canWrite()); + + _queued_write *q = (_queued_write *)linklistShift(&pending_writes); + + if (q == NULL) { + debugs(79, 3, HERE << this << " UFSStoreState::doWrite queue is empty"); + return; + } + + if (theFile->error()) { + debugs(79, DBG_IMPORTANT,HERE << "avoid write on theFile with error"); + debugs(79,3,HERE << "calling free_func for " << (void*) q->buf); + /* + * DPW 2006-05-24 + * Note "free_func" is memNodeWriteComplete(), which doesn't + * really free the memory. Instead it clears the node's + * write_pending flag. + */ + q->free_func((void*)q->buf); + delete q; + return; + } + + /* + * DPW 2006-05-24 + * UFSStoreState has a 'writing' flag that we used to set here, + * but it wasn't really used anywhere. In fact, some lower + * layers such as DISKD allow multiple outstanding writes, which + * makes the boolean writing flag meaningless. We would need + * a counter to keep track of writes going out and write callbacks + * coming in. For now let's just not use the writing flag at + * all. + */ + debugs(79, 3, HERE << this << " calling theFile->write(" << q->size << ")"); + + theFile->write(new WriteRequest(q->buf, q->offset, q->size, q->free_func)); + delete q; +} + +void +Fs::Ufs::UFSStoreState::readCompleted(const char *buf, int len, int errflag, RefCount result) +{ + assert (result.getRaw()); + reading = false; + debugs(79, 3, "UFSStoreState::readCompleted: dirno " << swap_dirn << + ", fileno "<< std::setfill('0') << std::hex << std::setw(8) << + swap_filen << " len "<< std::setfill(' ') << std::dec << len); + + if (len > 0) + offset_ += len; + + STRCB *callback_ = read.callback; + + assert(callback_); + + read.callback = NULL; + + void *cbdata; + + /* A note: + * diskd IO queues closes via the diskd queue. So close callbacks + * occur strictly after reads and writes. + * ufs doesn't queue, it simply completes, so close callbacks occur + * strictly after reads and writes. + * aufs performs closes syncronously, so close events must be managed + * to force strict ordering. + * The below does this: + * closing is set when theFile->close() has been called, and close only triggers + * when no io's are pending. + * writeCompleted likewise. + */ + if (!closing && cbdataReferenceValidDone(read.callback_data, &cbdata)) { + if (len > 0 && read_buf != buf) + memcpy(read_buf, buf, len); + + callback_(cbdata, read_buf, len, this); + } + + if (flags.try_closing || (theFile != NULL && theFile->error()) ) + tryClosing(); +} + +void +Fs::Ufs::UFSStoreState::writeCompleted(int errflag, size_t len, RefCount writeRequest) +{ + debugs(79, 3, HERE << "dirno " << swap_dirn << ", fileno " << + std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen << + ", len " << len); + /* + * DPW 2006-05-24 + * See doWrites() for why we don't update UFSStoreState::writing + * here anymore. + */ + + offset_ += len; + + if (theFile->error()) { + debugs(79,2,HERE << " detected an error, will try to close"); + tryClosing(); + } + + /* + * HNO 2009-07-24 + * Kick any pending write/close operations alive + */ + drainWriteQueue(); +} + +void +Fs::Ufs::UFSStoreState::doCloseCallback(int errflag) +{ + debugs(79, 3, "storeUfsIOCallback: errflag=" << errflag); + /* + * DPW 2006-05-24 + * When we signal the higher layer with this callback, it might unlock + * the StoreEntry and its associated data. We must "free" any queued + * I/Os (especially writes) now, otherwise the StoreEntry's mem_node's + * will have their write_pending flag set, and we'll get an assertion. + */ + freePending(); + STIOCB *theCallback = callback; + callback = NULL; + + void *cbdata; + + if (cbdataReferenceValidDone(callback_data, &cbdata) && theCallback) + theCallback(cbdata, errflag, this); + + /* + * We are finished with theFile since the lower layer signalled + * us that the file has been closed. This must be the last line, + * as theFile may be the only object holding us in memory. + */ + theFile = NULL; // refcounted +} + +/* ============= THE REAL UFS CODE ================ */ + +Fs::Ufs::UFSStoreState::UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callback_, void *callback_data_) : opening (false), creating (false), closing (false), reading(false), writing(false), pending_reads(NULL), pending_writes (NULL) +{ + swap_filen = anEntry->swap_filen; + swap_dirn = SD->index; + mode = O_BINARY; + callback = callback_; + callback_data = cbdataReference(callback_data_); + e = anEntry; + flags.write_draining = false; + flags.try_closing = false; +} + +Fs::Ufs::UFSStoreState::~UFSStoreState() +{ + assert(pending_reads == NULL); + assert(pending_writes == NULL); +} + +void +Fs::Ufs::UFSStoreState::freePending() +{ + _queued_read *qr; + + while ((qr = (_queued_read *)linklistShift(&pending_reads))) { + cbdataReferenceDone(qr->callback_data); + delete qr; + } + + debugs(79,3,HERE << "UFSStoreState::freePending: freed pending reads"); + + _queued_write *qw; + + while ((qw = (_queued_write *)linklistShift(&pending_writes))) { + if (qw->free_func) + qw->free_func(const_cast(qw->buf)); + delete qw; + } + + debugs(79,3,HERE << "UFSStoreState::freePending: freed pending writes"); +} + +bool +Fs::Ufs::UFSStoreState::kickReadQueue() +{ + _queued_read *q = (_queued_read *)linklistShift(&pending_reads); + + if (NULL == q) + return false; + + debugs(79, 3, "UFSStoreState::kickReadQueue: reading queued request of " << q->size << " bytes"); + + void *cbdata; + + if (cbdataReferenceValidDone(q->callback_data, &cbdata)) { + read_(q->buf, q->size, q->offset, q->callback, cbdata); + } else { + debugs(79, 2, "UFSStoreState::kickReadQueue: this: " << this << " cbdataReferenceValidDone returned false." << " closing: " << closing << " flags.try_closing: " << flags.try_closing); + delete q; + return false; + } + + delete q; + + return true; +} + +void +Fs::Ufs::UFSStoreState::queueRead(char *buf, size_t size, off_t aOffset, STRCB *callback_, void *callback_data_) +{ + debugs(79, 3, "UFSStoreState::queueRead: queueing read"); + assert(opening); + assert (pending_reads == NULL); + _queued_read *q = new _queued_read; + q->buf = buf; + q->size = size; + q->offset = aOffset; + q->callback = callback_; + q->callback_data = cbdataReference(callback_data_); + linklistPush(&pending_reads, q); +} + +/* + * DPW 2006-05-24 + * drainWriteQueue() is a loop around doWrite(). + */ +void +Fs::Ufs::UFSStoreState::drainWriteQueue() +{ + /* + * DPW 2007-04-12 + * We might find that flags.write_draining is already set + * because schemes like diskd can process I/O acks + * before sending another I/O request. e.g. the following + * sequence of events: open request -> write request -> + * drainWriteQueue() -> queue full -> callbacks -> openDone() -> + * drainWriteQueue(). + */ + if (flags.write_draining) + return; + + if (!theFile->canWrite()) + return; + + flags.write_draining = true; + + while (pending_writes != NULL) { + doWrite(); + } + + flags.write_draining = false; + + if (flags.try_closing) + tryClosing(); +} + +/* + * DPW 2006-05-24 + * This blows. DiskThreadsDiskFile::close() won't actually do the close + * if ioInProgress() is true. So we have to check it here. Maybe someday + * DiskThreadsDiskFile::close() will be modified to have a return value, + * or will remember to do the close for us. + */ +void +Fs::Ufs::UFSStoreState::tryClosing() +{ + debugs(79,3,HERE << this << " tryClosing()" << + " closing = " << closing << + " flags.try_closing = " << flags.try_closing << + " ioInProgress = " << theFile->ioInProgress()); + + if (theFile->ioInProgress()) { + debugs(79, 3, HERE << this << + " won't close since ioInProgress is true, bailing"); + flags.try_closing = true; + return; + } + + closing = true; + flags.try_closing = false; + theFile->close(); +} + +void +Fs::Ufs::UFSStoreState::queueWrite(char const *buf, size_t size, off_t aOffset, FREE * free_func) +{ + debugs(79, 3, HERE << this << " UFSStoreState::queueWrite: queueing write of size " << size); + + _queued_write *q; + q = new _queued_write; + q->buf = buf; + q->size = size; + q->offset = aOffset; + q->free_func = free_func; + linklistPush(&pending_writes, q); +} + === added file 'src/fs/ufs/UFSStoreState.h' --- src/fs/ufs/UFSStoreState.h 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/UFSStoreState.h 2012-08-10 06:56:49 +0000 @@ -0,0 +1,129 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_FS_UFS_UFSSTORESTATE_H +#define SQUID_FS_UFS_UFSSTORESTATE_H + +#include "DiskIO/IORequestor.h" +#include "StoreIOState.h" + +namespace Fs +{ +namespace Ufs +{ +/// \ingroup UFS +class UFSStoreState : public StoreIOState, public IORequestor +{ +public: + void * operator new (size_t); + void operator delete (void *); + UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callback_, void *callback_data_); + ~UFSStoreState(); + virtual void close(int how); + virtual void closeCompleted(); + // protected: + virtual void ioCompletedNotification(); + virtual void readCompleted(const char *buf, int len, int errflag, RefCount); + virtual void writeCompleted(int errflag, size_t len, RefCount); + RefCount theFile; + bool opening; + bool creating; + bool closing; + bool reading; + bool writing; + void read_(char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data); + void write(char const *buf, size_t size, off_t offset, FREE * free_func); + +protected: + virtual void doCloseCallback (int errflag); + + class _queued_read + { + public: + MEMPROXY_CLASS(UFSStoreState::_queued_read); + char *buf; + size_t size; + off_t offset; + STRCB *callback; + void *callback_data; + + }; + + class _queued_write + { + public: + MEMPROXY_CLASS(UFSStoreState::_queued_write); + char const *buf; + size_t size; + off_t offset; + FREE *free_func; + + }; + + /** \todo These should be in the IO strategy */ + + struct { + /** + * DPW 2006-05-24 + * the write_draining flag is used to avoid recursion inside + * the UFSStoreState::drainWriteQueue() method. + */ + bool write_draining; + /** + * DPW 2006-05-24 + * The try_closing flag is set by UFSStoreState::tryClosing() + * when UFSStoreState wants to close the file, but cannot + * because of pending I/Os. If set, UFSStoreState will + * try to close again in the I/O callbacks. + */ + bool try_closing; + } flags; + link_list *pending_reads; + link_list *pending_writes; + void queueRead(char *, size_t, off_t, STRCB *, void *); + void queueWrite(char const *, size_t, off_t, FREE *); + bool kickReadQueue(); + void drainWriteQueue(); + void tryClosing(); + char *read_buf; + +private: + void openDone(); + void freePending(); + void doWrite(); + CBDATA_CLASS(UFSStoreState); +}; + +MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_read); +MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_write); + +} //namespace Ufs +} //namespace Fs + +#endif /* SQUID_FS_UFS_UFSSTORESTATE_H */ === added file 'src/fs/ufs/UFSStrategy.cc' --- src/fs/ufs/UFSStrategy.cc 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/UFSStrategy.cc 2012-08-10 06:56:49 +0000 @@ -0,0 +1,183 @@ +/* + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + */ + +#include "squid.h" + +#include "DiskIO/DiskIOStrategy.h" +#include "UFSStrategy.h" +#include "UFSStoreState.h" +#include "UFSSwapDir.h" + +bool +Fs::Ufs::UFSStrategy::shedLoad() +{ + return io->shedLoad(); +} + +int +Fs::Ufs::UFSStrategy::load() +{ + return io->load(); +} + +Fs::Ufs::UFSStrategy::UFSStrategy (DiskIOStrategy *anIO) : io(anIO) +{} + +Fs::Ufs::UFSStrategy::~UFSStrategy () +{ + delete io; +} + +StoreIOState::Pointer +Fs::Ufs::UFSStrategy::createState(SwapDir *SD, StoreEntry *e, StoreIOState::STIOCB * aCallback, void *callback_data) const +{ + return new Fs::Ufs::UFSStoreState (SD, e, aCallback, callback_data); +} + +DiskFile::Pointer +Fs::Ufs::UFSStrategy::newFile (char const *path) +{ + return io->newFile(path); +} + +void +Fs::Ufs::UFSStrategy::unlinkFile(char const *path) +{ + io->unlinkFile(path); +} + +StoreIOState::Pointer +Fs::Ufs::UFSStrategy::open(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback, + StoreIOState::STIOCB * aCallback, void *callback_data) +{ + assert (((UFSSwapDir *)SD)->IO == this); + debugs(79, 3, HERE << "fileno "<< std::setfill('0') << std::hex + << std::uppercase << std::setw(8) << e->swap_filen); + + /* to consider: make createstate a private UFSStrategy call */ + StoreIOState::Pointer sio = createState (SD, e, aCallback, callback_data); + + sio->mode |= O_RDONLY; + + Fs::Ufs::UFSStoreState *state = dynamic_cast (sio.getRaw()); + + assert (state); + + char *path = ((UFSSwapDir *)SD)->fullPath(e->swap_filen, NULL); + + DiskFile::Pointer myFile = newFile (path); + + if (myFile.getRaw() == NULL) + return NULL; + + state->theFile = myFile; + + state->opening = true; + + myFile->open (sio->mode, 0644, state); + + if (myFile->error()) + return NULL; + + return sio; +} + +StoreIOState::Pointer +Fs::Ufs::UFSStrategy::create(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback, + StoreIOState::STIOCB * aCallback, void *callback_data) +{ + assert (((UFSSwapDir *)SD)->IO == this); + /* Allocate a number */ + sfileno filn = ((UFSSwapDir *)SD)->mapBitAllocate(); + debugs(79, 3, HERE << "fileno "<< std::setfill('0') << + std::hex << std::uppercase << std::setw(8) << filn); + + /* Shouldn't we handle a 'bitmap full' error here? */ + + StoreIOState::Pointer sio = createState (SD, e, aCallback, callback_data); + + sio->mode |= O_WRONLY | O_CREAT | O_TRUNC; + + sio->swap_filen = filn; + + Fs::Ufs::UFSStoreState *state = dynamic_cast (sio.getRaw()); + + assert (state); + + char *path = ((UFSSwapDir *)SD)->fullPath(filn, NULL); + + DiskFile::Pointer myFile = newFile (path); + + if (myFile.getRaw() == NULL) { + ((UFSSwapDir *)SD)->mapBitReset (filn); + return NULL; + } + + state->theFile = myFile; + + state->creating = true; + + myFile->create (state->mode, 0644, state); + + if (myFile->error()) { + ((UFSSwapDir *)SD)->mapBitReset (filn); + return NULL; + } + + /* now insert into the replacement policy */ + ((UFSSwapDir *)SD)->replacementAdd(e); + + return sio; +} + +int +Fs::Ufs::UFSStrategy::callback() +{ + return io->callback(); +} + +void +Fs::Ufs::UFSStrategy::init() +{ + io->init(); +} + +void +Fs::Ufs::UFSStrategy::sync() +{ + io->sync(); +} + +void +Fs::Ufs::UFSStrategy::statfs(StoreEntry & sentry)const +{ + io->statfs(sentry); +} === added file 'src/fs/ufs/UFSStrategy.h' --- src/fs/ufs/UFSStrategy.h 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/UFSStrategy.h 2012-08-10 06:56:49 +0000 @@ -0,0 +1,90 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_FS_UFS_UFSSTRATEGY_H +#define SQUID_FS_UFS_UFSSTRATEGY_H + +#include "DiskIO/DiskFile.h" +#include "StoreIOState.h" + +class Swapdir; +class StoreEntry; +class DiskIOStrategy; + +namespace Fs +{ +namespace Ufs +{ +/// \ingroup UFS +class UFSStrategy +{ +public: + UFSStrategy (DiskIOStrategy *); + virtual ~UFSStrategy (); + virtual bool shedLoad(); + + virtual int load(); + + StoreIOState::Pointer createState(SwapDir *SD, StoreEntry *e, StoreIOState::STIOCB * callback, void *callback_data) const; + /* UFS specific */ + virtual RefCount newFile (char const *path); + StoreIOState::Pointer open(SwapDir *, StoreEntry *, StoreIOState::STFNCB *, + StoreIOState::STIOCB *, void *); + StoreIOState::Pointer create(SwapDir *, StoreEntry *, StoreIOState::STFNCB *, + StoreIOState::STIOCB *, void *); + + virtual void unlinkFile (char const *); + virtual void sync(); + + virtual int callback(); + + /** Init per-instance logic */ + virtual void init(); + + /** cachemgr output on the IO instance stats */ + virtual void statfs(StoreEntry & sentry)const; + + /** The io strategy in use */ + DiskIOStrategy *io; + +protected: + + friend class UFSSwapDir; + +private: + UFSStrategy(); //disabled + UFSStrategy(UFSStrategy const &); //disabled + UFSStrategy &operator=(UFSStrategy const &); //disabled + +}; + +} //namespace Ufs +} //namespace Fs + +#endif /* SQUID_FS_UFS_UFSSTRATEGY_H */ === added file 'src/fs/ufs/UFSSwapDir.cc' --- src/fs/ufs/UFSSwapDir.cc 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/UFSSwapDir.cc 2012-08-10 06:56:49 +0000 @@ -0,0 +1,1350 @@ +/* + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + */ + +#include "squid-old.h" + +#define CLEAN_BUF_SZ 16384 + +#include "ConfigOption.h" +#include "DiskIO/DiskIOModule.h" +#include "FileMap.h" +#include "fde.h" +#include "Parsing.h" +#include "protos.h" +#include "RebuildState.h" +#include "SquidMath.h" +#include "DiskIO/DiskIOStrategy.h" +#include "StoreSearchUFS.h" +#include "StoreSwapLogData.h" +#include "SquidTime.h" +#include "StatCounters.h" +#include "UFSSwapDir.h" + + +int Fs::Ufs::UFSSwapDir::NumberOfUFSDirs = 0; +int *Fs::Ufs::UFSSwapDir::UFSDirToGlobalDirMapping = NULL; + +class UFSCleanLog : public SwapDir::CleanLog +{ + +public: + UFSCleanLog(SwapDir *); + /** Get the next entry that is a candidate for clean log writing + */ + virtual const StoreEntry *nextEntry(); + /** "write" an entry to the clean log file. + */ + virtual void write(StoreEntry const &); + char *cur; + char *newLog; + char *cln; + char *outbuf; + off_t outbuf_offset; + int fd; + RemovalPolicyWalker *walker; + SwapDir *sd; +}; + +UFSCleanLog::UFSCleanLog(SwapDir *aSwapDir) : + cur(NULL), newLog(NULL), cln(NULL), outbuf(NULL), + outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir) +{} + +const StoreEntry * +UFSCleanLog::nextEntry() +{ + const StoreEntry *entry = NULL; + + if (walker) + entry = walker->Next(walker); + + return entry; +} + +void +UFSCleanLog::write(StoreEntry const &e) +{ + StoreSwapLogData s; + static size_t ss = sizeof(StoreSwapLogData); + s.op = (char) SWAP_LOG_ADD; + s.swap_filen = e.swap_filen; + s.timestamp = e.timestamp; + s.lastref = e.lastref; + s.expires = e.expires; + s.lastmod = e.lastmod; + s.swap_file_sz = e.swap_file_sz; + s.refcount = e.refcount; + s.flags = e.flags; + memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH); + s.finalize(); + memcpy(outbuf + outbuf_offset, &s, ss); + outbuf_offset += ss; + /* buffered write */ + + if (outbuf_offset + ss >= CLEAN_BUF_SZ) { + if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) { + /* XXX This error handling should probably move up to the caller */ + debugs(50, DBG_CRITICAL, HERE << newLog << ": write: " << xstrerror()); + debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced."); + file_close(fd); + fd = -1; + unlink(newLog); + sd->cleanLog = NULL; + delete this; + return; + } + + outbuf_offset = 0; + } +} + +bool +Fs::Ufs::UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const +{ + if (!SwapDir::canStore(e, diskSpaceNeeded, load)) + return false; + + if (IO->shedLoad()) + return false; + + load = IO->load(); + return true; +} + +static void +FreeObject(void *address) +{ + StoreSwapLogData *anObject = static_cast (address); + delete anObject; +} + +static QS rev_int_sort; +static int +rev_int_sort(const void *A, const void *B) +{ + const int *i1 = (const int *)A; + const int *i2 = (const int *)B; + return *i2 - *i1; +} + +void +Fs::Ufs::UFSSwapDir::parseSizeL1L2() +{ + int i = GetInteger(); + if (i <= 0) + fatal("UFSSwapDir::parseSizeL1L2: invalid size value"); + + const uint64_t size = static_cast(i) << 20; // MBytes to Bytes + + /* just reconfigure it */ + if (reconfiguring) { + if (size == maxSize()) + debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB"); + else + debugs(3, DBG_IMPORTANT, "Cache dir '" << path << "' size changed to " << i << " MB"); + } + + max_size = size; + + l1 = GetInteger(); + + if (l1 <= 0) + fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value"); + + l2 = GetInteger(); + + if (l2 <= 0) + fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value"); +} + +void +Fs::Ufs::UFSSwapDir::reconfigure() +{ + parseSizeL1L2(); + parseOptions(1); +} + +void +Fs::Ufs::UFSSwapDir::parse (int anIndex, char *aPath) +{ + index = anIndex; + path = xstrdup(aPath); + + parseSizeL1L2(); + + /* Initialise replacement policy stuff */ + repl = createRemovalPolicy(Config.replPolicy); + + parseOptions(0); +} + +void +Fs::Ufs::UFSSwapDir::changeIO(DiskIOModule *module) +{ + DiskIOStrategy *anIO = module->createStrategy(); + safe_free(ioType); + ioType = xstrdup(module->type()); + + delete IO->io; + IO->io = anIO; + /* Change the IO Options */ + + if (currentIOOptions && currentIOOptions->options.size() > 2) + delete currentIOOptions->options.pop_back(); + + /* TODO: factor out these 4 lines */ + ConfigOption *ioOptions = IO->io->getOptionTree(); + + if (ioOptions) + currentIOOptions->options.push_back(ioOptions); +} + +bool +Fs::Ufs::UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig) +{ + if (strcmp(option, "IOEngine") != 0) + return false; + + if (isaReconfig) + /* silently ignore this */ + return true; + + if (!value) + self_destruct(); + + DiskIOModule *module = DiskIOModule::Find(value); + + if (!module) + self_destruct(); + + changeIO(module); + + return true; +} + +void +Fs::Ufs::UFSSwapDir::optionIODump(StoreEntry * e) const +{ + storeAppendPrintf(e, " IOEngine=%s", ioType); +} + +ConfigOption * +Fs::Ufs::UFSSwapDir::getOptionTree() const +{ + ConfigOption *parentResult = SwapDir::getOptionTree(); + + if (currentIOOptions == NULL) + currentIOOptions = new ConfigOptionVector(); + + currentIOOptions->options.push_back(parentResult); + + currentIOOptions->options.push_back(new ConfigOptionAdapter(*const_cast(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump)); + + if (ConfigOption *ioOptions = IO->io->getOptionTree()) + currentIOOptions->options.push_back(ioOptions); + + ConfigOption* result = currentIOOptions; + + currentIOOptions = NULL; + + return result; +} + +void +Fs::Ufs::UFSSwapDir::init() +{ + debugs(47, 3, HERE << "Initialising UFS SwapDir engine."); + /* Parsing must be finished by now - force to NULL, don't delete */ + currentIOOptions = NULL; + static int started_clean_event = 0; + static const char *errmsg = + "\tFailed to verify one of the swap directories, Check cache.log\n" + "\tfor details. Run 'squid -z' to create swap directories\n" + "\tif needed, or if running Squid for the first time."; + IO->init(); + + if (verifyCacheDirs()) + fatal(errmsg); + + openLog(); + + rebuild(); + + if (!started_clean_event) { + eventAdd("UFS storeDirClean", CleanEvent, NULL, 15.0, 1); + started_clean_event = 1; + } + + (void) storeDirGetBlkSize(path, &fs.blksize); +} + +void +Fs::Ufs::UFSSwapDir::create() +{ + debugs(47, 3, "Creating swap space in " << path); + createDirectory(path, 0); + createSwapSubDirs(); +} + +Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) : SwapDir(aType), IO(NULL), map(new FileMap()), suggest(0), swaplog_fd (-1), currentIOOptions(new ConfigOptionVector()), ioType(xstrdup(anIOType)), cur_size(0), n_disk_objects(0) +{ + /* modulename is only set to disk modules that are built, by configure, + * so the Find call should never return NULL here. + */ + IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy()); +} + +Fs::Ufs::UFSSwapDir::~UFSSwapDir() +{ + if (swaplog_fd > -1) { + file_close(swaplog_fd); + swaplog_fd = -1; + } + + delete map; + + if (IO) + delete IO; + + IO = NULL; + + safe_free(ioType); +} + +void +Fs::Ufs::UFSSwapDir::dumpEntry(StoreEntry &e) const +{ + debugs(47, DBG_CRITICAL, HERE << "FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); + debugs(47, DBG_CRITICAL, HERE << "PATH " << fullPath(e.swap_filen, NULL) ); + e.dump(0); +} + +bool +Fs::Ufs::UFSSwapDir::doubleCheck(StoreEntry & e) +{ + + struct stat sb; + + if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) { + debugs(47, DBG_CRITICAL, HERE << "WARNING: Missing swap file"); + dumpEntry(e); + return true; + } + + if ((off_t)e.swap_file_sz != sb.st_size) { + debugs(47, DBG_CRITICAL, HERE << "WARNING: Size Mismatch. Entry size: " + << e.swap_file_sz << ", file size: " << sb.st_size); + dumpEntry(e); + return true; + } + + return false; +} + +void +Fs::Ufs::UFSSwapDir::statfs(StoreEntry & sentry) const +{ + int totl_kb = 0; + int free_kb = 0; + int totl_in = 0; + int free_in = 0; + int x; + storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1); + storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2); + storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10); + storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0); + storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n", + Math::doublePercent(currentSize(), maxSize())); + storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n", + map->numFilesInMap(), map->capacity(), + Math::intPercent(map->numFilesInMap(), map->capacity())); + x = storeDirGetUFSStats(path, &totl_kb, &free_kb, &totl_in, &free_in); + + if (0 == x) { + storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", + totl_kb - free_kb, + totl_kb, + Math::intPercent(totl_kb - free_kb, totl_kb)); + storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", + totl_in - free_in, + totl_in, + Math::intPercent(totl_in - free_in, totl_in)); + } + + storeAppendPrintf(&sentry, "Flags:"); + + if (flags.selected) + storeAppendPrintf(&sentry, " SELECTED"); + + if (flags.read_only) + storeAppendPrintf(&sentry, " READ-ONLY"); + + storeAppendPrintf(&sentry, "\n"); + + IO->statfs(sentry); +} + +void +Fs::Ufs::UFSSwapDir::maintain() +{ + /* We can't delete objects while rebuilding swap */ + + /* XXX FIXME each store should start maintaining as it comes online. */ + + if (StoreController::store_dirs_rebuilding) + return; + + StoreEntry *e = NULL; + + int removed = 0; + + RemovalPurgeWalker *walker; + + double f = (double) (currentSize() - minSize()) / (maxSize() - minSize()); + + f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; + + int max_scan = (int) (f * 400.0 + 100.0); + + int max_remove = (int) (f * 70.0 + 10.0); + + /* + * This is kinda cheap, but so we need this priority hack? + */ + + debugs(47, 3, HERE << "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove ); + + walker = repl->PurgeInit(repl, max_scan); + + while (1) { + if (currentSize() < minSize()) + break; + + if (removed >= max_remove) + break; + + e = walker->Next(walker); + + if (!e) + break; /* no more objects */ + + ++removed; + + e->release(); + } + + walker->Done(walker); + debugs(47, (removed ? 2 : 3), HERE << path << + " removed " << removed << "/" << max_remove << " f=" << + std::setprecision(4) << f << " max_scan=" << max_scan); +} + +void +Fs::Ufs::UFSSwapDir::reference(StoreEntry &e) +{ + debugs(47, 3, HERE << "referencing " << &e << " " << + e.swap_dirn << "/" << e.swap_filen); + + if (repl->Referenced) + repl->Referenced(repl, &e, &e.repl); +} + +bool +Fs::Ufs::UFSSwapDir::dereference(StoreEntry & e) +{ + debugs(47, 3, HERE << "dereferencing " << &e << " " << + e.swap_dirn << "/" << e.swap_filen); + + if (repl->Dereferenced) + repl->Dereferenced(repl, &e, &e.repl); + + return true; // keep e in the global store_table +} + +StoreIOState::Pointer +Fs::Ufs::UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) +{ + return IO->create (this, &e, file_callback, aCallback, callback_data); +} + +StoreIOState::Pointer +Fs::Ufs::UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) +{ + return IO->open (this, &e, file_callback, aCallback, callback_data); +} + +int +Fs::Ufs::UFSSwapDir::mapBitTest(sfileno filn) +{ + return map->testBit(filn); +} + +void +Fs::Ufs::UFSSwapDir::mapBitSet(sfileno filn) +{ + map->setBit(filn); +} + +void +Fs::Ufs::UFSSwapDir::mapBitReset(sfileno filn) +{ + /* + * We have to test the bit before calling clearBit as + * it doesn't do bounds checking and blindly assumes + * filn is a valid file number, but it might not be because + * the map is dynamic in size. Also clearing an already clear + * bit puts the map counter of-of-whack. + */ + + if (map->testBit(filn)) + map->clearBit(filn); +} + +int +Fs::Ufs::UFSSwapDir::mapBitAllocate() +{ + int fn; + fn = map->allocate(suggest); + map->setBit(fn); + suggest = fn + 1; + return fn; +} + +char * +Fs::Ufs::UFSSwapDir::swapSubDir(int subdirn)const +{ + LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); + assert(0 <= subdirn && subdirn < l1); + snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn); + return fullfilename; +} + +int +Fs::Ufs::UFSSwapDir::createDirectory(const char *aPath, int should_exist) +{ + int created = 0; + + struct stat st; + getCurrentTime(); + + if (0 == ::stat(aPath, &st)) { + if (S_ISDIR(st.st_mode)) { + debugs(47, (should_exist ? 3 : DBG_IMPORTANT), aPath << " exists"); + } else { + fatalf("Swap directory %s is not a directory.", aPath); + } + +#if _SQUID_MSWIN_ + + } else if (0 == mkdir(aPath)) { +#else + + } else if (0 == mkdir(aPath, 0755)) { +#endif + debugs(47, (should_exist ? DBG_IMPORTANT : 3), aPath << " created"); + created = 1; + } else { + fatalf("Failed to make swap directory %s: %s", + aPath, xstrerror()); + } + + return created; +} + +bool +Fs::Ufs::UFSSwapDir::pathIsDirectory(const char *aPath)const +{ + + struct stat sb; + + if (::stat(aPath, &sb) < 0) { + debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerror()); + return false; + } + + if (S_ISDIR(sb.st_mode) == 0) { + debugs(47, DBG_CRITICAL, "WARNING: " << aPath << " is not a directory"); + return false; + } + + return true; +} + +bool +Fs::Ufs::UFSSwapDir::verifyCacheDirs() +{ + if (!pathIsDirectory(path)) + return true; + + for (int j = 0; j < l1; ++j) { + char const *aPath = swapSubDir(j); + + if (!pathIsDirectory(aPath)) + return true; + } + + return false; +} + +void +Fs::Ufs::UFSSwapDir::createSwapSubDirs() +{ + LOCAL_ARRAY(char, name, MAXPATHLEN); + + for (int i = 0; i < l1; ++i) { + snprintf(name, MAXPATHLEN, "%s/%02X", path, i); + + int should_exist; + + if (createDirectory(name, 0)) + should_exist = 0; + else + should_exist = 1; + + debugs(47, DBG_IMPORTANT, "Making directories in " << name); + + for (int k = 0; k < l2; ++k) { + snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k); + createDirectory(name, should_exist); + } + } +} + +char * +Fs::Ufs::UFSSwapDir::logFile(char const *ext) const +{ + LOCAL_ARRAY(char, lpath, MAXPATHLEN); + LOCAL_ARRAY(char, pathtmp, MAXPATHLEN); + LOCAL_ARRAY(char, digit, 32); + char *pathtmp2; + + if (Config.Log.swap) { + xstrncpy(pathtmp, path, MAXPATHLEN - 64); + pathtmp2 = pathtmp; + + while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL) + *pathtmp2 = '.'; + + while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') + pathtmp[strlen(pathtmp) - 1] = '\0'; + + for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2); + snprintf(lpath, MAXPATHLEN - 64, Config.Log.swap, pathtmp2); + + if (strncmp(lpath, Config.Log.swap, MAXPATHLEN - 64) == 0) { + strcat(lpath, "."); + snprintf(digit, 32, "%02d", index); + strncat(lpath, digit, 3); + } + } else { + xstrncpy(lpath, path, MAXPATHLEN - 64); + strcat(lpath, "/swap.state"); + } + + if (ext) + strncat(lpath, ext, 16); + + return lpath; +} + +void +Fs::Ufs::UFSSwapDir::openLog() +{ + char *logPath; + logPath = logFile(); + swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY); + + if (swaplog_fd < 0) { + debugs(50, DBG_IMPORTANT, "ERROR opening swap log " << logPath << ": " << xstrerror()); + fatal("UFSSwapDir::openLog: Failed to open swap log."); + } + + debugs(50, 3, HERE << "Cache Dir #" << index << " log opened on FD " << swaplog_fd); + + if (0 == NumberOfUFSDirs) + assert(NULL == UFSDirToGlobalDirMapping); + + ++NumberOfUFSDirs; + + assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured); +} + +void +Fs::Ufs::UFSSwapDir::closeLog() +{ + if (swaplog_fd < 0) /* not open */ + return; + + file_close(swaplog_fd); + + debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd); + + swaplog_fd = -1; + + --NumberOfUFSDirs; + + assert(NumberOfUFSDirs >= 0); + + if (0 == NumberOfUFSDirs) + safe_free(UFSDirToGlobalDirMapping); +} + +bool +Fs::Ufs::UFSSwapDir::validL1(int anInt) const +{ + return anInt < l1; +} + +bool +Fs::Ufs::UFSSwapDir::validL2(int anInt) const +{ + return anInt < l2; +} + +StoreEntry * +Fs::Ufs::UFSSwapDir::addDiskRestore(const cache_key * key, + sfileno file_number, + uint64_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + uint32_t refcount, + uint16_t newFlags, + int clean) +{ + StoreEntry *e = NULL; + debugs(47, 5, HERE << storeKeyText(key) << + ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number); + /* if you call this you'd better be sure file_number is not + * already in use! */ + e = new StoreEntry(); + e->store_status = STORE_OK; + e->setMemStatus(NOT_IN_MEMORY); + e->swap_status = SWAPOUT_DONE; + e->swap_filen = file_number; + e->swap_dirn = index; + e->swap_file_sz = swap_file_sz; + e->lock_count = 0; + e->lastref = lastref; + e->timestamp = timestamp; + e->expires = expires; + e->lastmod = lastmod; + e->refcount = refcount; + e->flags = newFlags; + EBIT_SET(e->flags, ENTRY_CACHABLE); + EBIT_CLR(e->flags, RELEASE_REQUEST); + EBIT_CLR(e->flags, KEY_PRIVATE); + e->ping_status = PING_NONE; + EBIT_CLR(e->flags, ENTRY_VALIDATED); + mapBitSet(e->swap_filen); + cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz); + ++n_disk_objects; + e->hashInsert(key); /* do it after we clear KEY_PRIVATE */ + replacementAdd (e); + return e; +} + +void +Fs::Ufs::UFSSwapDir::undoAddDiskRestore(StoreEntry *e) +{ + debugs(47, 5, HERE << *e); + replacementRemove(e); // checks swap_dirn so do it before we invalidate it + // Do not unlink the file as it might be used by a subsequent entry. + mapBitReset(e->swap_filen); + e->swap_filen = -1; + e->swap_dirn = -1; + cur_size -= fs.blksize * sizeInBlocks(e->swap_file_sz); + --n_disk_objects; +} + +void +Fs::Ufs::UFSSwapDir::rebuild() +{ + ++StoreController::store_dirs_rebuilding; + eventAdd("storeRebuild", Fs::Ufs::RebuildState::RebuildStep, new Fs::Ufs::RebuildState(this), 0.0, 1); +} + +void +Fs::Ufs::UFSSwapDir::closeTmpSwapLog() +{ + char *swaplog_path = xstrdup(logFile(NULL)); + char *new_path = xstrdup(logFile(".new")); + int fd; + file_close(swaplog_fd); + + if (xrename(new_path, swaplog_path) < 0) { + debugs(50, DBG_IMPORTANT, HERE << "ERROR: " << swaplog_path << ": " << xstrerror()); + fatalf("Failed to rename log file %s to %s.new", swaplog_path, swaplog_path); + } + + fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); + + if (fd < 0) { + debugs(50, DBG_IMPORTANT, HERE << "ERROR: " << swaplog_path << ": " << xstrerror()); + fatalf("Failed to open swap log %s", swaplog_path); + } + + safe_free(swaplog_path); + safe_free(new_path); + swaplog_fd = fd; + debugs(47, 3, HERE << "Cache Dir #" << index << " log opened on FD " << fd); +} + +FILE * +Fs::Ufs::UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) +{ + char *swaplog_path = xstrdup(logFile(NULL)); + char *clean_path = xstrdup(logFile(".last-clean")); + char *new_path = xstrdup(logFile(".new")); + + struct stat log_sb; + + struct stat clean_sb; + FILE *fp; + int fd; + + if (::stat(swaplog_path, &log_sb) < 0) { + debugs(47, DBG_IMPORTANT, HERE << "Cache Dir #" << index << ": No log file"); + safe_free(swaplog_path); + safe_free(clean_path); + safe_free(new_path); + return NULL; + } + + *zero_flag = log_sb.st_size == 0 ? 1 : 0; + /* close the existing write-only FD */ + + if (swaplog_fd >= 0) + file_close(swaplog_fd); + + /* open a write-only FD for the new log */ + fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); + + if (fd < 0) { + debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerror()); + fatal("UFSSwapDir::openTmpSwapLog: Failed to open swap log."); + } + + swaplog_fd = fd; + + { + const StoreSwapLogHeader header; + MemBuf buf; + buf.init(header.record_size, header.record_size); + buf.append(reinterpret_cast(&header), sizeof(header)); + // Pad to keep in sync with UFSSwapDir::writeCleanStart(). + memset(buf.space(), 0, header.gapSize()); + buf.appended(header.gapSize()); + file_write(swaplog_fd, -1, buf.content(), buf.contentSize(), + NULL, NULL, buf.freeFunc()); + } + + /* open a read-only stream of the old log */ + fp = fopen(swaplog_path, "rb"); + + if (fp == NULL) { + debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerror()); + fatal("Failed to open swap log for reading"); + } + + memset(&clean_sb, '\0', sizeof(struct stat)); + + if (::stat(clean_path, &clean_sb) < 0) + *clean_flag = 0; + else if (clean_sb.st_mtime < log_sb.st_mtime) + *clean_flag = 0; + else + *clean_flag = 1; + + safeunlink(clean_path, 1); + + safe_free(swaplog_path); + + safe_free(clean_path); + + safe_free(new_path); + + return fp; +} + +/* + * Begin the process to write clean cache state. For AUFS this means + * opening some log files and allocating write buffers. Return 0 if + * we succeed, and assign the 'func' and 'data' return pointers. + */ +int +Fs::Ufs::UFSSwapDir::writeCleanStart() +{ + UFSCleanLog *state = new UFSCleanLog(this); + StoreSwapLogHeader header; +#if HAVE_FCHMOD + + struct stat sb; +#endif + + cleanLog = NULL; + state->newLog = xstrdup(logFile(".clean")); + state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); + + if (state->fd < 0) { + xfree(state->newLog); + delete state; + return -1; + } + + state->cur = xstrdup(logFile(NULL)); + state->cln = xstrdup(logFile(".last-clean")); + state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1); + state->outbuf_offset = 0; + /*copy the header */ + memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader)); + // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog(). + memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize()); + state->outbuf_offset += header.record_size; + + state->walker = repl->WalkInit(repl); + ::unlink(state->cln); + debugs(47, 3, HERE << "opened " << state->newLog << ", FD " << state->fd); +#if HAVE_FCHMOD + + if (::stat(state->cur, &sb) == 0) + fchmod(state->fd, sb.st_mode); + +#endif + + + cleanLog = state; + return 0; +} + +void +Fs::Ufs::UFSSwapDir::writeCleanDone() +{ + UFSCleanLog *state = (UFSCleanLog *)cleanLog; + int fd; + + if (NULL == state) + return; + + if (state->fd < 0) + return; + + state->walker->Done(state->walker); + + if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { + debugs(50, DBG_CRITICAL, HERE << state->newLog << ": write: " << xstrerror()); + debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced."); + file_close(state->fd); + state->fd = -1; + ::unlink(state->newLog); + } + + safe_free(state->outbuf); + /* + * You can't rename open files on Microsoft "operating systems" + * so we have to close before renaming. + */ + closeLog(); + /* save the fd value for a later test */ + fd = state->fd; + /* rename */ + + if (state->fd >= 0) { +#if _SQUID_OS2_ || _SQUID_WINDOWS_ + file_close(state->fd); + state->fd = -1; +#endif + + xrename(state->newLog, state->cur); + } + + /* touch a timestamp file if we're not still validating */ + if (StoreController::store_dirs_rebuilding) + (void) 0; + else if (fd < 0) + (void) 0; + else + file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY)); + + /* close */ + safe_free(state->cur); + + safe_free(state->newLog); + + safe_free(state->cln); + + if (state->fd >= 0) + file_close(state->fd); + + state->fd = -1; + + delete state; + + cleanLog = NULL; +} + +void +Fs::Ufs::UFSSwapDir::CleanEvent(void *unused) +{ + static int swap_index = 0; + int i; + int j = 0; + int n = 0; + /* + * Assert that there are UFS cache_dirs configured, otherwise + * we should never be called. + */ + assert(NumberOfUFSDirs); + + if (NULL == UFSDirToGlobalDirMapping) { + SwapDir *sd; + /* + * Initialize the little array that translates UFS cache_dir + * number into the Config.cacheSwap.swapDirs array index. + */ + UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping)); + + for (i = 0, n = 0; i < Config.cacheSwap.n_configured; ++i) { + /* This is bogus, the controller should just clean each instance once */ + sd = dynamic_cast (INDEXSD(i)); + + if (!UFSSwapDir::IsUFSDir(sd)) + continue; + + UFSSwapDir *usd = dynamic_cast(sd); + + assert (usd); + + UFSDirToGlobalDirMapping[n] = i; + ++n; + + j += (usd->l1 * usd->l2); + } + + assert(n == NumberOfUFSDirs); + /* + * Start the commonUfsDirClean() swap_index with a random + * value. j equals the total number of UFS level 2 + * swap directories + */ + swap_index = (int) (squid_random() % j); + } + + /* if the rebuild is finished, start cleaning directories. */ + if (0 == StoreController::store_dirs_rebuilding) { + n = DirClean(swap_index); + ++swap_index; + } + + eventAdd("storeDirClean", CleanEvent, NULL, + 15.0 * exp(-0.25 * n), 1); +} + +bool +Fs::Ufs::UFSSwapDir::IsUFSDir(SwapDir * sd) +{ + UFSSwapDir *mySD = dynamic_cast(sd); + return (mySD != 0) ; +} + +/* + * XXX: this is broken - it assumes all cache dirs use the same + * l1 and l2 scheme. -RBC 20021215. Partial fix is in place - + * if not UFSSwapDir return 0; + */ +bool +Fs::Ufs::UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2) +{ + int D1, D2; + int L1, L2; + int filn = fn; + assert(F0 < Config.cacheSwap.n_configured); + assert (UFSSwapDir::IsUFSDir (dynamic_cast(INDEXSD(F0)))); + UFSSwapDir *sd = dynamic_cast(INDEXSD(F0)); + + if (!sd) + return 0; + + L1 = sd->l1; + + L2 = sd->l2; + + D1 = ((filn / L2) / L2) % L1; + + if (F1 != D1) + return 0; + + D2 = (filn / L2) % L2; + + if (F2 != D2) + return 0; + + return 1; +} + +int +Fs::Ufs::UFSSwapDir::validFileno(sfileno filn, int flag) const +{ + if (filn < 0) + return 0; + + /* + * If flag is set it means out-of-range file number should + * be considered invalid. + */ + if (flag) + if (filn > map->capacity()) + return 0; + + return 1; +} + +void +Fs::Ufs::UFSSwapDir::unlinkFile(sfileno f) +{ + debugs(79, 3, HERE << "unlinking fileno " << std::setfill('0') << + std::hex << std::uppercase << std::setw(8) << f << " '" << + fullPath(f,NULL) << "'"); + /* commonUfsDirMapBitReset(this, f); */ + IO->unlinkFile(fullPath(f,NULL)); +} + +bool +Fs::Ufs::UFSSwapDir::unlinkdUseful() const +{ + // unlinkd may be useful only in workers + return IamWorkerProcess() && IO->io->unlinkdUseful(); +} + +void +Fs::Ufs::UFSSwapDir::unlink(StoreEntry & e) +{ + debugs(79, 3, HERE << "dirno " << index << ", fileno "<< + std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); + if (e.swap_status == SWAPOUT_DONE) { + cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); + --n_disk_objects; + } + replacementRemove(&e); + mapBitReset(e.swap_filen); + UFSSwapDir::unlinkFile(e.swap_filen); +} + +void +Fs::Ufs::UFSSwapDir::replacementAdd(StoreEntry * e) +{ + debugs(47, 4, HERE << "added node " << e << " to dir " << index); + repl->Add(repl, e, &e->repl); +} + + +void +Fs::Ufs::UFSSwapDir::replacementRemove(StoreEntry * e) +{ + StorePointer SD; + + if (e->swap_dirn < 0) + return; + + SD = INDEXSD(e->swap_dirn); + + assert (dynamic_cast(SD.getRaw()) == this); + + debugs(47, 4, HERE << "remove node " << e << " from dir " << index); + + repl->Remove(repl, e, &e->repl); +} + +void +Fs::Ufs::UFSSwapDir::dump(StoreEntry & entry) const +{ + storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2); + dumpOptions(&entry); +} + +char * +Fs::Ufs::UFSSwapDir::fullPath(sfileno filn, char *fullpath) const +{ + LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); + int L1 = l1; + int L2 = l2; + + if (!fullpath) + fullpath = fullfilename; + + fullpath[0] = '\0'; + + snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X", + path, + ((filn / L2) / L2) % L1, + (filn / L2) % L2, + filn); + + return fullpath; +} + +int +Fs::Ufs::UFSSwapDir::callback() +{ + return IO->callback(); +} + +void +Fs::Ufs::UFSSwapDir::sync() +{ + IO->sync(); +} + +void +Fs::Ufs::UFSSwapDir::swappedOut(const StoreEntry &e) +{ + cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz); + ++n_disk_objects; +} + +StoreSearch * +Fs::Ufs::UFSSwapDir::search(String const url, HttpRequest *request) +{ + if (url.size()) + fatal ("Cannot search by url yet\n"); + + return new Fs::Ufs::StoreSearchUFS (this); +} + +void +Fs::Ufs::UFSSwapDir::logEntry(const StoreEntry & e, int op) const +{ + StoreSwapLogData *s = new StoreSwapLogData; + s->op = (char) op; + s->swap_filen = e.swap_filen; + s->timestamp = e.timestamp; + s->lastref = e.lastref; + s->expires = e.expires; + s->lastmod = e.lastmod; + s->swap_file_sz = e.swap_file_sz; + s->refcount = e.refcount; + s->flags = e.flags; + memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH); + s->finalize(); + file_write(swaplog_fd, + -1, + s, + sizeof(StoreSwapLogData), + NULL, + NULL, + FreeObject); +} + +int +Fs::Ufs::UFSSwapDir::DirClean(int swap_index) +{ + DIR *dir_pointer = NULL; + + LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); + LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); + + int files[20]; + int swapfileno; + int fn; /* same as swapfileno, but with dirn bits set */ + int n = 0; + int k = 0; + int N0, N1, N2; + int D0, D1, D2; + UFSSwapDir *SD; + N0 = NumberOfUFSDirs; + D0 = UFSDirToGlobalDirMapping[swap_index % N0]; + SD = dynamic_cast(INDEXSD(D0)); + assert (SD); + N1 = SD->l1; + D1 = (swap_index / N0) % N1; + N2 = SD->l2; + D2 = ((swap_index / N0) / N1) % N2; + snprintf(p1, MAXPATHLEN, "%s/%02X/%02X", + SD->path, D1, D2); + debugs(36, 3, HERE << "Cleaning directory " << p1); + dir_pointer = opendir(p1); + + if (dir_pointer == NULL) { + if (errno == ENOENT) { + debugs(36, DBG_CRITICAL, HERE << "WARNING: Creating " << p1); +#if _SQUID_MSWIN_ + + if (mkdir(p1) == 0) +#else + + if (mkdir(p1, 0777) == 0) +#endif + + return 0; + } + + debugs(50, DBG_CRITICAL, HERE << p1 << ": " << xstrerror()); + safeunlink(p1, 1); + return 0; + } + + dirent_t *de; + while ((de = readdir(dir_pointer)) != NULL && k < 20) { + if (sscanf(de->d_name, "%X", &swapfileno) != 1) + continue; + + fn = swapfileno; /* XXX should remove this cruft ! */ + + if (SD->validFileno(fn, 1)) + if (SD->mapBitTest(fn)) + if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2)) + continue; + + files[k] = swapfileno; + ++k; + } + + closedir(dir_pointer); + + if (k == 0) + return 0; + + qsort(files, k, sizeof(int), rev_int_sort); + + if (k > 10) + k = 10; + + for (n = 0; n < k; ++n) { + debugs(36, 3, HERE << "Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]); + snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); + safeunlink(p2, 0); + ++statCounter.swap.files_cleaned; + } + + debugs(36, 3, HERE << "Cleaned " << k << " unused files from " << p1); + return k; +} === added file 'src/fs/ufs/UFSSwapDir.h' --- src/fs/ufs/UFSSwapDir.h 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/UFSSwapDir.h 2012-08-10 06:56:49 +0000 @@ -0,0 +1,191 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + */ + +#ifndef SQUID_FS_UFS_UFSSWAPDIR_H +#define SQUID_FS_UFS_UFSSWAPDIR_H + +#include "SquidString.h" +#include "Store.h" +#include "StoreIOState.h" +#include "StoreSearch.h" +#include "SwapDir.h" +#include "swap_log_op.h" +#include "UFSStrategy.h" + +class HttpRequest; +class ConfigOptionVector; +class FileMap; +class DiskIOModule; + +namespace Fs +{ +namespace Ufs +{ +/// \ingroup UFS +class UFSSwapDir : public SwapDir +{ +public: + static bool IsUFSDir(SwapDir* sd); + static int DirClean(int swap_index); + /** check whether swapfile belongs to the specified cachedir/l1dir/l2dir + * + * \param cachedir the number of the cachedir which is being tested + * \param level1dir level-1 dir in the cachedir + * \param level2dir level-2 dir + */ + static bool FilenoBelongsHere(int fn, int cachedir, int level1dir, int level2dir); + + UFSSwapDir(char const *aType, const char *aModuleType); + /** Initial setup / end destruction */ + virtual void init(); + /** Create a new SwapDir (-z command-line option) */ + virtual void create(); + virtual void dump(StoreEntry &) const; + ~UFSSwapDir(); + virtual StoreSearch *search(String const url, HttpRequest *); + /** double-check swap during rebuild (-S command-line option) + * + * called by storeCleanup if needed + */ + virtual bool doubleCheck(StoreEntry &); + virtual bool unlinkdUseful() const; + /** unlink a file, and remove its entry from the filemap */ + virtual void unlink(StoreEntry &); + virtual void statfs(StoreEntry &)const; + virtual void maintain(); + /** check whether this filesystem can store the given object + * + * UFS filesystems will happily store anything as long as + * the LRU time isn't too small + */ + virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const; + /** reference an object + * + * This routine is called whenever an object is referenced, so we can + * maintain replacement information within the storage fs. + */ + virtual void reference(StoreEntry &); + /** de-reference an object + * + * This routine is called whenever the last reference to an object is + * removed, to maintain replacement information within the storage fs. + */ + virtual bool dereference(StoreEntry &); + virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); + virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); + virtual void openLog(); + virtual void closeLog(); + virtual int writeCleanStart(); + virtual void writeCleanDone(); + virtual void logEntry(const StoreEntry & e, int op) const; + virtual void parse(int index, char *path); ///parse configuration and setup new SwapDir + virtual void reconfigure(); ///reconfigure the SwapDir + virtual int callback(); + virtual void sync(); + virtual void swappedOut(const StoreEntry &e); + virtual uint64_t currentSize() const { return cur_size; } + virtual uint64_t currentCount() const { return n_disk_objects; } + + void unlinkFile(sfileno f); + // move down when unlink is a virtual method + //protected: + Fs::Ufs::UFSStrategy *IO; + char *fullPath(sfileno, char *) const; + /* temp */ + void closeTmpSwapLog(); + FILE *openTmpSwapLog(int *clean_flag, int *zero_flag); + char *swapSubDir(int subdirn) const; + int mapBitTest(sfileno filn); + void mapBitReset(sfileno filn); + void mapBitSet(sfileno filn); + /** Add a new object to the cache with empty memory copy and pointer to disk + * + * This method is used to rebuild a store from disk + */ + StoreEntry *addDiskRestore(const cache_key * key, + sfileno file_number, + uint64_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + uint32_t refcount, + uint16_t flags, + int clean); + /// Undo the effects of UFSSwapDir::addDiskRestore(). + void undoAddDiskRestore(StoreEntry *e); + int validFileno(sfileno filn, int flag) const; + int mapBitAllocate(); + virtual ConfigOption *getOptionTree() const; + + void *fsdata; + + bool validL2(int) const; + bool validL1(int) const; + + /** Add and remove the given StoreEntry from the replacement policy in use */ + void replacementAdd(StoreEntry *e); + void replacementRemove(StoreEntry *e); + +protected: + FileMap *map; + int suggest; + int l1; + int l2; + +private: + void parseSizeL1L2(); + static int NumberOfUFSDirs; + static int * UFSDirToGlobalDirMapping; + bool pathIsDirectory(const char *path)const; + int swaplog_fd; + static EVH CleanEvent; + /** Verify that the the CacheDir exists + * + * If this returns < 0, then Squid exits, complains about swap + * directories not existing, and instructs the admin to run 'squid -z' + * Called by UFSSwapDir::init + */ + bool verifyCacheDirs(); + void rebuild(); + int createDirectory(const char *path, int); + void createSwapSubDirs(); + void dumpEntry(StoreEntry &) const; + char *logFile(char const *ext = NULL)const; + void changeIO(DiskIOModule *); + bool optionIOParse(char const *option, const char *value, int reconfiguring); + void optionIODump(StoreEntry * e) const; + mutable ConfigOptionVector *currentIOOptions; + char const *ioType; + uint64_t cur_size; ///< currently used space in the storage area + uint64_t n_disk_objects; ///< total number of objects stored +}; + +} //namespace Ufs +} //namespace Fs +#endif /* SQUID_FS_UFS_UFSSWAPDIR_H */ === added file 'src/fs/ufs/UFSSwapLogParser.cc' --- src/fs/ufs/UFSSwapLogParser.cc 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/UFSSwapLogParser.cc 2012-08-10 06:56:49 +0000 @@ -0,0 +1,199 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "md5.h" +#include "StoreSwapLogData.h" +#include "swap_log_op.h" +#include "UFSSwapLogParser.h" + +/// Parse a swap header entry created on a system with 32-bit size_t and sfileno +/// this is typical of 32-bit systems without large file support +/// NP: SQUID_MD5_DIGEST_LENGTH is very risky still. +class UFSSwapLogParser_v1_32bs:public Fs::Ufs::UFSSwapLogParser +{ +public: + /// version 1 cache swap.state entry with 32-bit size_t (swap_file_sz) + /// time_t an sfileno have no variation from the v1 baseline format + struct StoreSwapLogDataOld { + char op; + sfileno swap_filen; + time_t timestamp; + time_t lastref; + time_t expires; + time_t lastmod; + uint32_t swap_file_sz; + uint16_t refcount; + uint16_t flags; + unsigned char key[SQUID_MD5_DIGEST_LENGTH]; + }; + UFSSwapLogParser_v1_32bs(FILE *fp):Fs::Ufs::UFSSwapLogParser(fp) { + record_size = sizeof(UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld); + } + /// Convert the on-disk 32-bit format to our current format while reading + bool ReadRecord(StoreSwapLogData &swapData) { + UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld readData; + int bytes = sizeof(UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld); + + assert(log); + + if (fread(&readData, bytes, 1, log) != 1) { + return false; + } + swapData.op = readData.op; + swapData.swap_filen = readData.swap_filen; + swapData.timestamp = readData.timestamp; + swapData.lastref = readData.lastref; + swapData.expires = readData.expires; + swapData.lastmod = readData.lastmod; + swapData.swap_file_sz = readData.swap_file_sz; + swapData.refcount = readData.refcount; + swapData.flags = readData.flags; + memcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); + return true; + } +}; + +/// swap.state v2 log parser +class UFSSwapLogParser_v2: public Fs::Ufs::UFSSwapLogParser +{ +public: + UFSSwapLogParser_v2(FILE *fp): Fs::Ufs::UFSSwapLogParser(fp) { + record_size = sizeof(StoreSwapLogData); + } + bool ReadRecord(StoreSwapLogData &swapData) { + assert(log); + return fread(&swapData, sizeof(StoreSwapLogData), 1, log) == 1; + } +}; + +Fs::Ufs::UFSSwapLogParser * +Fs::Ufs::UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp) +{ + StoreSwapLogHeader header; + + assert(fp); + + if (fread(&header, sizeof(StoreSwapLogHeader), 1, fp) != 1) + return NULL; + + if (header.op != SWAP_LOG_VERSION) { + debugs(47, DBG_IMPORTANT, "Old swap file detected..."); + fseek(fp, 0, SEEK_SET); + return new UFSSwapLogParser_v1_32bs(fp); // Um. 32-bits except time_t, and can't determine that. + } + + debugs(47, 2, "Swap file version: " << header.version); + + if (header.version == 1) { + if (fseek(fp, header.record_size, SEEK_SET) != 0) + return NULL; + + debugs(47, DBG_IMPORTANT, "Rejecting swap file v1 to avoid cache " << + "index corruption. Forcing a full cache index rebuild. " << + "See Squid bug #3441."); + return NULL; + +#if UNUSED_CODE + // baseline + // 32-bit sfileno + // native time_t (hopefully 64-bit) + // 64-bit file size + if (header.record_size == sizeof(StoreSwapLogData)) { + debugs(47, DBG_IMPORTANT, "Version 1 of swap file with LFS support detected... "); + return new UFSSwapLogParser_v1(fp); + } + + // which means we have a 3-way grid of permutations to import (yuck!) + // 1) sfileno 32-bit / 64-bit (64-bit was broken) + // 2) time_t 32-bit / 64-bit + // 3) size_t 32-bit / 64-bit (32-bit was pre-LFS) + + // 32-bit systems... + // only LFS (size_t) differs from baseline + if (header.record_size == sizeof(struct UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld)) { + debugs(47, DBG_IMPORTANT, "Version 1 (32-bit) swap file without LFS support detected... "); + return new UFSSwapLogParser_v1_32bs(fp); + } + // LFS (size_t) and timestamps (time_t) differs from baseline + if (header.record_size == sizeof(struct UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld)) { + debugs(47, DBG_IMPORTANT, "Version 1 (32-bit) swap file with short timestamps and without LFS support detected... "); + return new UFSSwapLogParser_v1_32bst(fp); + } + // No downgrade for 64-bit timestamps to 32-bit. + + // 64-bit systems + // sfileno was 64-bit for a some builds + if (header.record_size == sizeof(struct UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld)) { + debugs(47, DBG_IMPORTANT, "Version 1 (64-bit) swap file with broken sfileno detected... "); + return new UFSSwapLogParser_v1_64bfn(fp); + } + // NP: 64-bit system with 32-bit size_t/time_t are not handled. + + debugs(47, DBG_IMPORTANT, "WARNING: The swap file has wrong format!... "); + debugs(47, DBG_IMPORTANT, "NOTE: Cannot safely downgrade caches to short (32-bit) timestamps."); + return NULL; +#endif + } + + if (header.version >= 2) { + if (!header.sane()) { + debugs(47, DBG_IMPORTANT, "ERROR: Corrupted v" << header.version << + " swap file header."); + return NULL; + } + + if (fseek(fp, header.record_size, SEEK_SET) != 0) + return NULL; + + if (header.version == 2) + return new UFSSwapLogParser_v2(fp); + } + + // TODO: v3: write to disk in network-order bytes for the larger fields? + + debugs(47, DBG_IMPORTANT, "Unknown swap file version: " << header.version); + return NULL; +} + +int +Fs::Ufs::UFSSwapLogParser::SwapLogEntries() +{ + struct stat sb; + + if (log_entries >= 0) + return log_entries; + + if (log && record_size && 0 == fstat(fileno(log), &sb)) { + log_entries = sb.st_size/record_size; + return log_entries; + } + + return 0; +} === added file 'src/fs/ufs/UFSSwapLogParser.h' --- src/fs/ufs/UFSSwapLogParser.h 1970-01-01 00:00:00 +0000 +++ src/fs/ufs/UFSSwapLogParser.h 2012-08-10 06:56:49 +0000 @@ -0,0 +1,67 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_FS_UFS_UFSSWAPLOGPARSER_H +#define SQUID_FS_UFS_UFSSWAPLOGPARSER_H + +#include + +class StoreSwapLogData; + +namespace Fs +{ +namespace Ufs +{ +/// \ingroup UFS +class UFSSwapLogParser +{ +public: + FILE *log; + int log_entries; + int record_size; + + UFSSwapLogParser(FILE *fp):log(fp),log_entries(-1), record_size(0) { + } + virtual ~UFSSwapLogParser() {}; + + static UFSSwapLogParser *GetUFSSwapLogParser(FILE *fp); + + virtual bool ReadRecord(StoreSwapLogData &swapData) = 0; + int SwapLogEntries(); + void Close() { + if (log) { + fclose(log); + log = NULL; + } + } +}; + +} //namespace Ufs +} //namespace Fs +#endif /* SQUID_FS_UFS_UFSSWAPLOGPARSER_H */ === removed file 'src/fs/ufs/store_dir_ufs.cc' --- src/fs/ufs/store_dir_ufs.cc 2012-08-08 07:21:13 +0000 +++ src/fs/ufs/store_dir_ufs.cc 1970-01-01 00:00:00 +0000 @@ -1,1457 +0,0 @@ - -/* - * $Id$ - * - * DEBUG: section 47 Store Directory Routines - * AUTHOR: Duane Wessels - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid-old.h" -#include "Store.h" -#include "fde.h" -#include "ufscommon.h" -#include "StoreSwapLogData.h" -#include "ConfigOption.h" -#include "DiskIO/DiskIOStrategy.h" -#include "DiskIO/DiskIOModule.h" -#include "FileMap.h" -#include "Parsing.h" -#include "SquidMath.h" -#include "SquidTime.h" -#include "StatCounters.h" -#include "SwapDir.h" -#include "swap_log_op.h" - -int UFSSwapDir::NumberOfUFSDirs = 0; -int *UFSSwapDir::UFSDirToGlobalDirMapping = NULL; - -/* - * storeUfsDirCheckObj - * - * This routine is called by storeDirSelectSwapDir to see if the given - * object is able to be stored on this filesystem. UFS filesystems will - * happily store anything as long as the LRU time isn't too small. - */ -bool -UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const -{ - if (!SwapDir::canStore(e, diskSpaceNeeded, load)) - return false; - - if (IO->shedLoad()) - return false; - - load = IO->load(); - return true; -} - - -/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ - -void -UFSSwapDir::parseSizeL1L2() -{ - int i = GetInteger(); - if (i <= 0) - fatal("UFSSwapDir::parseSizeL1L2: invalid size value"); - - const uint64_t size = static_cast(i) << 20; // MBytes to Bytes - - /* just reconfigure it */ - if (reconfiguring) { - if (size == maxSize()) - debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB"); - else - debugs(3, 1, "Cache dir '" << path << "' size changed to " << i << " MB"); - } - - max_size = size; - - l1 = GetInteger(); - - if (l1 <= 0) - fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value"); - - l2 = GetInteger(); - - if (l2 <= 0) - fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value"); -} - -/* - * storeUfsDirReconfigure - * - * This routine is called when the given swapdir needs reconfiguring - */ - -void -UFSSwapDir::reconfigure() -{ - parseSizeL1L2(); - parseOptions(1); -} - -/* - * storeUfsDirParse - * - * Called when a *new* fs is being setup. - */ -void -UFSSwapDir::parse (int anIndex, char *aPath) -{ - index = anIndex; - path = xstrdup(aPath); - - parseSizeL1L2(); - - /* Initialise replacement policy stuff */ - repl = createRemovalPolicy(Config.replPolicy); - - parseOptions(0); -} - -void -UFSSwapDir::changeIO(DiskIOModule *module) -{ - DiskIOStrategy *anIO = module->createStrategy(); - safe_free(ioType); - ioType = xstrdup(module->type()); - - delete IO->io; - IO->io = anIO; - /* Change the IO Options */ - - if (currentIOOptions && currentIOOptions->options.size() > 2) - delete currentIOOptions->options.pop_back(); - - /* TODO: factor out these 4 lines */ - ConfigOption *ioOptions = IO->io->getOptionTree(); - - if (ioOptions) - currentIOOptions->options.push_back(ioOptions); -} - -bool -UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig) -{ - if (strcmp(option, "IOEngine") != 0) - return false; - - if (isaReconfig) - /* silently ignore this */ - return true; - - if (!value) - self_destruct(); - - DiskIOModule *module = DiskIOModule::Find(value); - - if (!module) - self_destruct(); - - changeIO(module); - - return true; -} - -void -UFSSwapDir::optionIODump(StoreEntry * e) const -{ - storeAppendPrintf(e, " IOEngine=%s", ioType); -} - -ConfigOption * -UFSSwapDir::getOptionTree() const -{ - ConfigOption *parentResult = SwapDir::getOptionTree(); - - if (currentIOOptions == NULL) - currentIOOptions = new ConfigOptionVector(); - - currentIOOptions->options.push_back(parentResult); - - currentIOOptions->options.push_back(new ConfigOptionAdapter(*const_cast(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump)); - - if (ConfigOption *ioOptions = IO->io->getOptionTree()) - currentIOOptions->options.push_back(ioOptions); - - ConfigOption* result = currentIOOptions; - - currentIOOptions = NULL; - - return result; -} - -/* - * Initial setup / end destruction - */ -void -UFSSwapDir::init() -{ - debugs(47, 3, "Initialising UFS SwapDir engine."); - /* Parsing must be finished by now - force to NULL, don't delete */ - currentIOOptions = NULL; - static int started_clean_event = 0; - static const char *errmsg = - "\tFailed to verify one of the swap directories, Check cache.log\n" - "\tfor details. Run 'squid -z' to create swap directories\n" - "\tif needed, or if running Squid for the first time."; - IO->init(); - - if (verifyCacheDirs()) - fatal(errmsg); - - openLog(); - - rebuild(); - - if (!started_clean_event) { - eventAdd("UFS storeDirClean", CleanEvent, NULL, 15.0, 1); - started_clean_event = 1; - } - - (void) storeDirGetBlkSize(path, &fs.blksize); -} - -void -UFSSwapDir::create() -{ - debugs(47, 3, "Creating swap space in " << path); - createDirectory(path, 0); - createSwapSubDirs(); -} - -UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) : SwapDir(aType), IO(NULL), map(new FileMap()), suggest(0), swaplog_fd (-1), currentIOOptions(new ConfigOptionVector()), ioType(xstrdup(anIOType)), cur_size(0), n_disk_objects(0) -{ - /* modulename is only set to disk modules that are built, by configure, - * so the Find call should never return NULL here. - */ - IO = new UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy()); -} - -UFSSwapDir::~UFSSwapDir() -{ - if (swaplog_fd > -1) { - file_close(swaplog_fd); - swaplog_fd = -1; - } - - delete map; - - if (IO) - delete IO; - - IO = NULL; - - safe_free(ioType); -} - -void -UFSSwapDir::dumpEntry(StoreEntry &e) const -{ - debugs(47, 0, "UFSSwapDir::dumpEntry: FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); - debugs(47, 0, "UFSSwapDir::dumpEntry: PATH " << fullPath(e.swap_filen, NULL) ); - e.dump(0); -} - -/* - * UFSSwapDir::doubleCheck - * - * This is called by storeCleanup() if -S was given on the command line. - */ -bool -UFSSwapDir::doubleCheck(StoreEntry & e) -{ - - struct stat sb; - - if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) { - debugs(47, 0, "UFSSwapDir::doubleCheck: MISSING SWAP FILE"); - dumpEntry(e); - return true; - } - - if ((off_t)e.swap_file_sz != sb.st_size) { - debugs(47, 0, "UFSSwapDir::doubleCheck: SIZE MISMATCH"); - debugs(47, 0, "UFSSwapDir::doubleCheck: ENTRY SIZE: " << e.swap_file_sz << ", FILE SIZE: " << sb.st_size); - dumpEntry(e); - return true; - } - - return false; -} - -void -UFSSwapDir::statfs(StoreEntry & sentry) const -{ - int totl_kb = 0; - int free_kb = 0; - int totl_in = 0; - int free_in = 0; - int x; - storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1); - storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2); - storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10); - storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0); - storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n", - Math::doublePercent(currentSize(), maxSize())); - storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n", - map->numFilesInMap(), map->capacity(), - Math::intPercent(map->numFilesInMap(), map->capacity())); - x = storeDirGetUFSStats(path, &totl_kb, &free_kb, &totl_in, &free_in); - - if (0 == x) { - storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", - totl_kb - free_kb, - totl_kb, - Math::intPercent(totl_kb - free_kb, totl_kb)); - storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", - totl_in - free_in, - totl_in, - Math::intPercent(totl_in - free_in, totl_in)); - } - - storeAppendPrintf(&sentry, "Flags:"); - - if (flags.selected) - storeAppendPrintf(&sentry, " SELECTED"); - - if (flags.read_only) - storeAppendPrintf(&sentry, " READ-ONLY"); - - storeAppendPrintf(&sentry, "\n"); - - IO->statfs(sentry); -} - -void -UFSSwapDir::maintain() -{ - /* We can't delete objects while rebuilding swap */ - - /* XXX FIXME each store should start maintaining as it comes online. */ - - if (StoreController::store_dirs_rebuilding) - return; - - StoreEntry *e = NULL; - - int removed = 0; - - RemovalPurgeWalker *walker; - - double f = (double) (currentSize() - minSize()) / (maxSize() - minSize()); - - f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; - - int max_scan = (int) (f * 400.0 + 100.0); - - int max_remove = (int) (f * 70.0 + 10.0); - - /* - * This is kinda cheap, but so we need this priority hack? - */ - - debugs(47, 3, "storeMaintainSwapSpace: f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove ); - - walker = repl->PurgeInit(repl, max_scan); - - while (1) { - if (currentSize() < minSize()) - break; - - if (removed >= max_remove) - break; - - e = walker->Next(walker); - - if (!e) - break; /* no more objects */ - - ++removed; - - e->release(); - } - - walker->Done(walker); - debugs(47, (removed ? 2 : 3), "UFSSwapDir::maintain: " << path << - " removed " << removed << "/" << max_remove << " f=" << - std::setprecision(4) << f << " max_scan=" << max_scan); -} - -/* - * UFSSwapDir::reference - * - * This routine is called whenever an object is referenced, so we can - * maintain replacement information within the storage fs. - */ -void -UFSSwapDir::reference(StoreEntry &e) -{ - debugs(47, 3, "UFSSwapDir::reference: referencing " << &e << " " << e.swap_dirn << "/" << e.swap_filen); - - if (repl->Referenced) - repl->Referenced(repl, &e, &e.repl); -} - -/* - * UFSSwapDir::dereference - * This routine is called whenever the last reference to an object is - * removed, to maintain replacement information within the storage fs. - */ -bool -UFSSwapDir::dereference(StoreEntry & e) -{ - debugs(47, 3, "UFSSwapDir::dereference: referencing " << &e << " " << e.swap_dirn << "/" << e.swap_filen); - - if (repl->Dereferenced) - repl->Dereferenced(repl, &e, &e.repl); - - return true; // keep e in the global store_table -} - -StoreIOState::Pointer -UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) -{ - return IO->create (this, &e, file_callback, aCallback, callback_data); -} - -StoreIOState::Pointer -UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) -{ - return IO->open (this, &e, file_callback, aCallback, callback_data); -} - -int -UFSSwapDir::mapBitTest(sfileno filn) -{ - return map->testBit(filn); -} - -void -UFSSwapDir::mapBitSet(sfileno filn) -{ - map->setBit(filn); -} - -void -UFSSwapDir::mapBitReset(sfileno filn) -{ - /* - * We have to test the bit before calling clearBit as - * it doesn't do bounds checking and blindly assumes - * filn is a valid file number, but it might not be because - * the map is dynamic in size. Also clearing an already clear - * bit puts the map counter of-of-whack. - */ - - if (map->testBit(filn)) - map->clearBit(filn); -} - -int -UFSSwapDir::mapBitAllocate() -{ - int fn; - fn = map->allocate(suggest); - map->setBit(fn); - suggest = fn + 1; - return fn; -} - -char * -UFSSwapDir::swapSubDir(int subdirn)const -{ - LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); - assert(0 <= subdirn && subdirn < l1); - snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn); - return fullfilename; -} - -int -UFSSwapDir::createDirectory(const char *aPath, int should_exist) -{ - int created = 0; - - struct stat st; - getCurrentTime(); - - if (0 == ::stat(aPath, &st)) { - if (S_ISDIR(st.st_mode)) { - debugs(47, (should_exist ? 3 : 1), aPath << " exists"); - } else { - fatalf("Swap directory %s is not a directory.", aPath); - } - } else if (0 == mkdir(aPath, 0755)) { - debugs(47, (should_exist ? 1 : 3), aPath << " created"); - created = 1; - } else { - fatalf("Failed to make swap directory %s: %s", - aPath, xstrerror()); - } - - return created; -} - -bool -UFSSwapDir::pathIsDirectory(const char *aPath)const -{ - - struct stat sb; - - if (::stat(aPath, &sb) < 0) { - debugs(47, 0, "" << aPath << ": " << xstrerror()); - return false; - } - - if (S_ISDIR(sb.st_mode) == 0) { - debugs(47, 0, "" << aPath << " is not a directory"); - return false; - } - - return true; -} - -/* - * This function is called by commonUfsDirInit(). If this returns < 0, - * then Squid exits, complains about swap directories not - * existing, and instructs the admin to run 'squid -z' - */ -bool -UFSSwapDir::verifyCacheDirs() -{ - if (!pathIsDirectory(path)) - return true; - - for (int j = 0; j < l1; ++j) { - char const *aPath = swapSubDir(j); - - if (!pathIsDirectory(aPath)) - return true; - } - - return false; -} - -void -UFSSwapDir::createSwapSubDirs() -{ - LOCAL_ARRAY(char, name, MAXPATHLEN); - - for (int i = 0; i < l1; ++i) { - snprintf(name, MAXPATHLEN, "%s/%02X", path, i); - - int should_exist; - - if (createDirectory(name, 0)) - should_exist = 0; - else - should_exist = 1; - - debugs(47, 1, "Making directories in " << name); - - for (int k = 0; k < l2; ++k) { - snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k); - createDirectory(name, should_exist); - } - } -} - -char * -UFSSwapDir::logFile(char const *ext) const -{ - LOCAL_ARRAY(char, lpath, MAXPATHLEN); - LOCAL_ARRAY(char, pathtmp, MAXPATHLEN); - LOCAL_ARRAY(char, digit, 32); - char *pathtmp2; - - if (Config.Log.swap) { - xstrncpy(pathtmp, path, MAXPATHLEN - 64); - pathtmp2 = pathtmp; - - while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL) - *pathtmp2 = '.'; - - while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') - pathtmp[strlen(pathtmp) - 1] = '\0'; - - for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2); - snprintf(lpath, MAXPATHLEN - 64, Config.Log.swap, pathtmp2); - - if (strncmp(lpath, Config.Log.swap, MAXPATHLEN - 64) == 0) { - strcat(lpath, "."); - snprintf(digit, 32, "%02d", index); - strncat(lpath, digit, 3); - } - } else { - xstrncpy(lpath, path, MAXPATHLEN - 64); - strcat(lpath, "/swap.state"); - } - - if (ext) - strncat(lpath, ext, 16); - - return lpath; -} - -void -UFSSwapDir::openLog() -{ - char *logPath; - logPath = logFile(); - swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY); - - if (swaplog_fd < 0) { - debugs(50, 1, "" << logPath << ": " << xstrerror()); - fatal("commonUfsDirOpenSwapLog: Failed to open swap log."); - } - - debugs(50, 3, "Cache Dir #" << index << " log opened on FD " << swaplog_fd); - - if (0 == NumberOfUFSDirs) - assert(NULL == UFSDirToGlobalDirMapping); - - ++NumberOfUFSDirs; - - assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured); -} - -void -UFSSwapDir::closeLog() -{ - if (swaplog_fd < 0) /* not open */ - return; - - file_close(swaplog_fd); - - debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd); - - swaplog_fd = -1; - - --NumberOfUFSDirs; - - assert(NumberOfUFSDirs >= 0); - - if (0 == NumberOfUFSDirs) - safe_free(UFSDirToGlobalDirMapping); -} - -bool -UFSSwapDir::validL1(int anInt) const -{ - return anInt < l1; -} - -bool -UFSSwapDir::validL2(int anInt) const -{ - return anInt < l2; -} - -/* Add a new object to the cache with empty memory copy and pointer to disk - * use to rebuild store from disk. */ -StoreEntry * -UFSSwapDir::addDiskRestore(const cache_key * key, - sfileno file_number, - uint64_t swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - uint32_t refcount, - uint16_t newFlags, - int clean) -{ - StoreEntry *e = NULL; - debugs(47, 5, "commonUfsAddDiskRestore: " << storeKeyText(key) << - ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number); - /* if you call this you'd better be sure file_number is not - * already in use! */ - e = new StoreEntry(); - e->store_status = STORE_OK; - e->setMemStatus(NOT_IN_MEMORY); - e->swap_status = SWAPOUT_DONE; - e->swap_filen = file_number; - e->swap_dirn = index; - e->swap_file_sz = swap_file_sz; - e->lock_count = 0; - e->lastref = lastref; - e->timestamp = timestamp; - e->expires = expires; - e->lastmod = lastmod; - e->refcount = refcount; - e->flags = newFlags; - EBIT_SET(e->flags, ENTRY_CACHABLE); - EBIT_CLR(e->flags, RELEASE_REQUEST); - EBIT_CLR(e->flags, KEY_PRIVATE); - e->ping_status = PING_NONE; - EBIT_CLR(e->flags, ENTRY_VALIDATED); - mapBitSet(e->swap_filen); - cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz); - ++n_disk_objects; - e->hashInsert(key); /* do it after we clear KEY_PRIVATE */ - replacementAdd (e); - return e; -} - -void -UFSSwapDir::undoAddDiskRestore(StoreEntry *e) -{ - debugs(47, 5, HERE << *e); - replacementRemove(e); // checks swap_dirn so do it before we invalidate it - // Do not unlink the file as it might be used by a subsequent entry. - mapBitReset(e->swap_filen); - e->swap_filen = -1; - e->swap_dirn = -1; - cur_size -= fs.blksize * sizeInBlocks(e->swap_file_sz); - --n_disk_objects; -} - -void -UFSSwapDir::rebuild() -{ - ++StoreController::store_dirs_rebuilding; - eventAdd("storeRebuild", RebuildState::RebuildStep, new RebuildState(this), 0.0, 1); -} - -void -UFSSwapDir::closeTmpSwapLog() -{ - char *swaplog_path = xstrdup(logFile(NULL)); - char *new_path = xstrdup(logFile(".new")); - int fd; - file_close(swaplog_fd); - - if (xrename(new_path, swaplog_path) < 0) { - debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror()); - fatalf("Failed to rename log file %s to %s.new", swaplog_path, swaplog_path); - } - - fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); - - if (fd < 0) { - debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror()); - fatalf("Failed to open swap log %s", swaplog_path); - } - - safe_free(swaplog_path); - safe_free(new_path); - swaplog_fd = fd; - debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd); -} - -FILE * -UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) -{ - char *swaplog_path = xstrdup(logFile(NULL)); - char *clean_path = xstrdup(logFile(".last-clean")); - char *new_path = xstrdup(logFile(".new")); - - struct stat log_sb; - - struct stat clean_sb; - FILE *fp; - int fd; - - if (::stat(swaplog_path, &log_sb) < 0) { - debugs(47, 1, "Cache Dir #" << index << ": No log file"); - safe_free(swaplog_path); - safe_free(clean_path); - safe_free(new_path); - return NULL; - } - - *zero_flag = log_sb.st_size == 0 ? 1 : 0; - /* close the existing write-only FD */ - - if (swaplog_fd >= 0) - file_close(swaplog_fd); - - /* open a write-only FD for the new log */ - fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - - if (fd < 0) { - debugs(50, 1, "" << new_path << ": " << xstrerror()); - fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); - } - - swaplog_fd = fd; - - { - const StoreSwapLogHeader header; - MemBuf buf; - buf.init(header.record_size, header.record_size); - buf.append(reinterpret_cast(&header), sizeof(header)); - // Pad to keep in sync with UFSSwapDir::writeCleanStart(). - memset(buf.space(), 0, header.gapSize()); - buf.appended(header.gapSize()); - file_write(swaplog_fd, -1, buf.content(), buf.contentSize(), - NULL, NULL, buf.freeFunc()); - } - - /* open a read-only stream of the old log */ - fp = fopen(swaplog_path, "rb"); - - if (fp == NULL) { - debugs(50, 0, "" << swaplog_path << ": " << xstrerror()); - fatal("Failed to open swap log for reading"); - } - - memset(&clean_sb, '\0', sizeof(struct stat)); - - if (::stat(clean_path, &clean_sb) < 0) - *clean_flag = 0; - else if (clean_sb.st_mtime < log_sb.st_mtime) - *clean_flag = 0; - else - *clean_flag = 1; - - safeunlink(clean_path, 1); - - safe_free(swaplog_path); - - safe_free(clean_path); - - safe_free(new_path); - - return fp; -} - -class UFSCleanLog : public SwapDir::CleanLog -{ - -public: - UFSCleanLog(SwapDir *); - virtual const StoreEntry *nextEntry(); - virtual void write(StoreEntry const &); - char *cur; - char *newLog; - char *cln; - char *outbuf; - off_t outbuf_offset; - int fd; - RemovalPolicyWalker *walker; - SwapDir *sd; -}; - -#define CLEAN_BUF_SZ 16384 - - -UFSCleanLog::UFSCleanLog(SwapDir *aSwapDir) : cur(NULL),newLog(NULL),cln(NULL),outbuf(NULL), - outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir) -{} - -/* - * Begin the process to write clean cache state. For AUFS this means - * opening some log files and allocating write buffers. Return 0 if - * we succeed, and assign the 'func' and 'data' return pointers. - */ -int -UFSSwapDir::writeCleanStart() -{ - UFSCleanLog *state = new UFSCleanLog(this); - StoreSwapLogHeader header; -#if HAVE_FCHMOD - - struct stat sb; -#endif - - cleanLog = NULL; - state->newLog = xstrdup(logFile(".clean")); - state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - - if (state->fd < 0) { - xfree(state->newLog); - delete state; - return -1; - } - - state->cur = xstrdup(logFile(NULL)); - state->cln = xstrdup(logFile(".last-clean")); - state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1); - state->outbuf_offset = 0; - /*copy the header */ - memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader)); - // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog(). - memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize()); - state->outbuf_offset += header.record_size; - - state->walker = repl->WalkInit(repl); - ::unlink(state->cln); - debugs(47, 3, "storeDirWriteCleanLogs: opened " << state->newLog << ", FD " << state->fd); -#if HAVE_FCHMOD - - if (::stat(state->cur, &sb) == 0) - fchmod(state->fd, sb.st_mode); - -#endif - - - cleanLog = state; - return 0; -} - -/* - * Get the next entry that is a candidate for clean log writing - */ -const StoreEntry * -UFSCleanLog::nextEntry() -{ - const StoreEntry *entry = NULL; - - if (walker) - entry = walker->Next(walker); - - return entry; -} - -/* - * "write" an entry to the clean log file. - */ -void -UFSCleanLog::write(StoreEntry const &e) -{ - StoreSwapLogData s; - static size_t ss = sizeof(StoreSwapLogData); - s.op = (char) SWAP_LOG_ADD; - s.swap_filen = e.swap_filen; - s.timestamp = e.timestamp; - s.lastref = e.lastref; - s.expires = e.expires; - s.lastmod = e.lastmod; - s.swap_file_sz = e.swap_file_sz; - s.refcount = e.refcount; - s.flags = e.flags; - memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH); - s.finalize(); - memcpy(outbuf + outbuf_offset, &s, ss); - outbuf_offset += ss; - /* buffered write */ - - if (outbuf_offset + ss >= CLEAN_BUF_SZ) { - if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) { - /* XXX This error handling should probably move up to the caller */ - debugs(50, 0, "storeDirWriteCleanLogs: " << newLog << ": write: " << xstrerror()); - debugs(50, 0, "storeDirWriteCleanLogs: Current swap logfile not replaced."); - file_close(fd); - fd = -1; - unlink(newLog); - sd->cleanLog = NULL; - delete this; - return; - } - - outbuf_offset = 0; - } -} - -void -UFSSwapDir::writeCleanDone() -{ - UFSCleanLog *state = (UFSCleanLog *)cleanLog; - int fd; - - if (NULL == state) - return; - - if (state->fd < 0) - return; - - state->walker->Done(state->walker); - - if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { - debugs(50, 0, "storeDirWriteCleanLogs: " << state->newLog << ": write: " << xstrerror()); - debugs(50, 0, "storeDirWriteCleanLogs: Current swap logfile not replaced."); - file_close(state->fd); - state->fd = -1; - ::unlink(state->newLog); - } - - safe_free(state->outbuf); - /* - * You can't rename open files on Microsoft "operating systems" - * so we have to close before renaming. - */ - closeLog(); - /* save the fd value for a later test */ - fd = state->fd; - /* rename */ - - if (state->fd >= 0) { -#if _SQUID_OS2_ || _SQUID_WINDOWS_ - file_close(state->fd); - state->fd = -1; -#endif - - xrename(state->newLog, state->cur); - } - - /* touch a timestamp file if we're not still validating */ - if (StoreController::store_dirs_rebuilding) - (void) 0; - else if (fd < 0) - (void) 0; - else - file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY)); - - /* close */ - safe_free(state->cur); - - safe_free(state->newLog); - - safe_free(state->cln); - - if (state->fd >= 0) - file_close(state->fd); - - state->fd = -1; - - delete state; - - cleanLog = NULL; -} - -static void -FreeObject(void *address) -{ - StoreSwapLogData *anObject = static_cast (address); - delete anObject; -} - -void -UFSSwapDir::logEntry(const StoreEntry & e, int op) const -{ - StoreSwapLogData *s = new StoreSwapLogData; - s->op = (char) op; - s->swap_filen = e.swap_filen; - s->timestamp = e.timestamp; - s->lastref = e.lastref; - s->expires = e.expires; - s->lastmod = e.lastmod; - s->swap_file_sz = e.swap_file_sz; - s->refcount = e.refcount; - s->flags = e.flags; - memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH); - s->finalize(); - file_write(swaplog_fd, - -1, - s, - sizeof(StoreSwapLogData), - NULL, - NULL, - FreeObject); -} - -static QS rev_int_sort; -static int -rev_int_sort(const void *A, const void *B) -{ - const int *i1 = (const int *)A; - const int *i2 = (const int *)B; - return *i2 - *i1; -} - -int -UFSSwapDir::DirClean(int swap_index) -{ - DIR *dir_pointer = NULL; - - LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); - LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); - - int files[20]; - int swapfileno; - int fn; /* same as swapfileno, but with dirn bits set */ - int n = 0; - int k = 0; - int N0, N1, N2; - int D0, D1, D2; - UFSSwapDir *SD; - N0 = NumberOfUFSDirs; - D0 = UFSDirToGlobalDirMapping[swap_index % N0]; - SD = dynamic_cast(INDEXSD(D0)); - assert (SD); - N1 = SD->l1; - D1 = (swap_index / N0) % N1; - N2 = SD->l2; - D2 = ((swap_index / N0) / N1) % N2; - snprintf(p1, MAXPATHLEN, "%s/%02X/%02X", - SD->path, D1, D2); - debugs(36, 3, "storeDirClean: Cleaning directory " << p1); - dir_pointer = opendir(p1); - - if (dir_pointer == NULL) { - if (errno == ENOENT) { - debugs(36, 0, "storeDirClean: WARNING: Creating " << p1); - if (mkdir(p1, 0777) == 0) - return 0; - } - - debugs(50, 0, "storeDirClean: " << p1 << ": " << xstrerror()); - safeunlink(p1, 1); - return 0; - } - - dirent_t *de; - while ((de = readdir(dir_pointer)) != NULL && k < 20) { - if (sscanf(de->d_name, "%X", &swapfileno) != 1) - continue; - - fn = swapfileno; /* XXX should remove this cruft ! */ - - if (SD->validFileno(fn, 1)) - if (SD->mapBitTest(fn)) - if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2)) - continue; - - files[k] = swapfileno; - ++k; - } - - closedir(dir_pointer); - - if (k == 0) - return 0; - - qsort(files, k, sizeof(int), rev_int_sort); - - if (k > 10) - k = 10; - - for (n = 0; n < k; ++n) { - debugs(36, 3, "storeDirClean: Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]); - snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); - safeunlink(p2, 0); - ++statCounter.swap.files_cleaned; - } - - debugs(36, 3, "Cleaned " << k << " unused files from " << p1); - return k; -} - -void -UFSSwapDir::CleanEvent(void *unused) -{ - static int swap_index = 0; - int i; - int j = 0; - int n = 0; - /* - * Assert that there are UFS cache_dirs configured, otherwise - * we should never be called. - */ - assert(NumberOfUFSDirs); - - if (NULL == UFSDirToGlobalDirMapping) { - SwapDir *sd; - /* - * Initialize the little array that translates UFS cache_dir - * number into the Config.cacheSwap.swapDirs array index. - */ - UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping)); - - for (i = 0, n = 0; i < Config.cacheSwap.n_configured; ++i) { - /* This is bogus, the controller should just clean each instance once */ - sd = dynamic_cast (INDEXSD(i)); - - if (!UFSSwapDir::IsUFSDir(sd)) - continue; - - UFSSwapDir *usd = dynamic_cast(sd); - - assert (usd); - - UFSDirToGlobalDirMapping[n] = i; - ++n; - - j += (usd->l1 * usd->l2); - } - - assert(n == NumberOfUFSDirs); - /* - * Start the commonUfsDirClean() swap_index with a random - * value. j equals the total number of UFS level 2 - * swap directories - */ - swap_index = (int) (squid_random() % j); - } - - /* if the rebuild is finished, start cleaning directories. */ - if (0 == StoreController::store_dirs_rebuilding) { - n = DirClean(swap_index); - ++swap_index; - } - - eventAdd("storeDirClean", CleanEvent, NULL, - 15.0 * exp(-0.25 * n), 1); -} - -int -UFSSwapDir::IsUFSDir(SwapDir * sd) -{ - UFSSwapDir *mySD = dynamic_cast(sd); - return mySD ? 1 : 0 ; -} - -/* - * Does swapfile number 'fn' belong in cachedir #F0, - * level1 dir #F1, level2 dir #F2? - * XXX: this is broken - it assumes all cache dirs use the same - * l1 and l2 scheme. -RBC 20021215. Partial fix is in place - - * if not UFSSwapDir return 0; - */ -int -UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2) -{ - int D1, D2; - int L1, L2; - int filn = fn; - assert(F0 < Config.cacheSwap.n_configured); - assert (UFSSwapDir::IsUFSDir (dynamic_cast(INDEXSD(F0)))); - UFSSwapDir *sd = dynamic_cast(INDEXSD(F0)); - - if (!sd) - return 0; - - L1 = sd->l1; - - L2 = sd->l2; - - D1 = ((filn / L2) / L2) % L1; - - if (F1 != D1) - return 0; - - D2 = (filn / L2) % L2; - - if (F2 != D2) - return 0; - - return 1; -} - - -int -UFSSwapDir::validFileno(sfileno filn, int flag) const -{ - if (filn < 0) - return 0; - - /* - * If flag is set it means out-of-range file number should - * be considered invalid. - */ - if (flag) - if (filn > map->capacity()) - return 0; - - return 1; -} - - - -/* - * UFSSwapDir::unlinkFile - * - * This routine unlinks a file and pulls it out of the bitmap. - * It used to be in commonUfsUnlink(), however an interface change - * forced this bit of code here. Eeek. - */ -void -UFSSwapDir::unlinkFile(sfileno f) -{ - debugs(79, 3, "UFSSwapDir::unlinkFile: unlinking fileno " << std::setfill('0') << - std::hex << std::uppercase << std::setw(8) << f << " '" << - fullPath(f,NULL) << "'"); - /* commonUfsDirMapBitReset(this, f); */ - IO->unlinkFile(fullPath(f,NULL)); -} - -bool -UFSSwapDir::unlinkdUseful() const -{ - // unlinkd may be useful only in workers - return IamWorkerProcess() && IO->io->unlinkdUseful(); -} - -void -UFSSwapDir::unlink(StoreEntry & e) -{ - debugs(79, 3, "storeUfsUnlink: dirno " << index << ", fileno "<< - std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); - if (e.swap_status == SWAPOUT_DONE) { - cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); - --n_disk_objects; - } - replacementRemove(&e); - mapBitReset(e.swap_filen); - UFSSwapDir::unlinkFile(e.swap_filen); -} - -/* - * Add and remove the given StoreEntry from the replacement policy in - * use. - */ - -void -UFSSwapDir::replacementAdd(StoreEntry * e) -{ - debugs(47, 4, "UFSSwapDir::replacementAdd: added node " << e << " to dir " << index); - repl->Add(repl, e, &e->repl); -} - - -void -UFSSwapDir::replacementRemove(StoreEntry * e) -{ - StorePointer SD; - - if (e->swap_dirn < 0) - return; - - SD = INDEXSD(e->swap_dirn); - - assert (dynamic_cast(SD.getRaw()) == this); - - debugs(47, 4, "UFSSwapDir::replacementRemove: remove node " << e << " from dir " << index); - - repl->Remove(repl, e, &e->repl); -} - -void -UFSSwapDir::dump(StoreEntry & entry) const -{ - storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2); - dumpOptions(&entry); -} - -char * -UFSSwapDir::fullPath(sfileno filn, char *fullpath) const -{ - LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); - int L1 = l1; - int L2 = l2; - - if (!fullpath) - fullpath = fullfilename; - - fullpath[0] = '\0'; - - snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X", - path, - ((filn / L2) / L2) % L1, - (filn / L2) % L2, - filn); - - return fullpath; -} - -int -UFSSwapDir::callback() -{ - return IO->callback(); -} - -void -UFSSwapDir::sync() -{ - IO->sync(); -} - -void -UFSSwapDir::swappedOut(const StoreEntry &e) -{ - cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz); - ++n_disk_objects; -} - -StoreSearch * -UFSSwapDir::search(String const url, HttpRequest *request) -{ - if (url.size()) - fatal ("Cannot search by url yet\n"); - - return new StoreSearchUFS (this); -} - -CBDATA_CLASS_INIT(StoreSearchUFS); -StoreSearchUFS::StoreSearchUFS(RefCount aSwapDir) : sd(aSwapDir), walker (sd->repl->WalkInit(sd->repl)), current (NULL), _done (false) -{} - -/* do not link -StoreSearchUFS::StoreSearchUFS(StoreSearchUFS const &); -*/ - -StoreSearchUFS::~StoreSearchUFS() -{ - walker->Done(walker); - walker = NULL; -} - -void -StoreSearchUFS::next(void (aCallback)(void *cbdata), void *aCallbackArgs) -{ - next(); - aCallback(aCallbackArgs); -} - -bool -StoreSearchUFS::next() -{ - /* the walker API doesn't make sense. the store entries referred to are already readwrite - * from their hash table entries - */ - - if (walker) - current = const_cast(walker->Next(walker)); - - if (current == NULL) - _done = true; - - return current != NULL; -} - -bool -StoreSearchUFS::error() const -{ - return false; -} - -bool -StoreSearchUFS::isDone() const -{ - return _done; -} - -StoreEntry * -StoreSearchUFS::currentItem() -{ - return current; -} === removed file 'src/fs/ufs/store_io_ufs.cc' --- src/fs/ufs/store_io_ufs.cc 2012-08-06 17:41:08 +0000 +++ src/fs/ufs/store_io_ufs.cc 1970-01-01 00:00:00 +0000 @@ -1,662 +0,0 @@ - -/* - * $Id$ - * - * DEBUG: section 79 Storage Manager UFS Interface - * AUTHOR: Duane Wessels - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid-old.h" -#include "Store.h" -#include "ufscommon.h" -#include "Generic.h" -#include "DiskIO/DiskFile.h" -#include "DiskIO/DiskIOStrategy.h" -#include "DiskIO/ReadRequest.h" -#include "DiskIO/WriteRequest.h" - -#include "SwapDir.h" - -bool -UFSStrategy::shedLoad() -{ - return io->shedLoad(); -} - -int -UFSStrategy::load() -{ - return io->load(); -} - -UFSStrategy::UFSStrategy (DiskIOStrategy *anIO) : io(anIO) -{} - -UFSStrategy::~UFSStrategy () -{ - delete io; -} - -StoreIOState::Pointer -UFSStrategy::createState(SwapDir *SD, StoreEntry *e, StoreIOState::STIOCB * aCallback, void *callback_data) const -{ - return new UFSStoreState (SD, e, aCallback, callback_data); -} - -DiskFile::Pointer -UFSStrategy::newFile (char const *path) -{ - return io->newFile(path); -} - - -void -UFSStrategy::unlinkFile(char const *path) -{ - io->unlinkFile(path); -} - -CBDATA_CLASS_INIT(UFSStoreState); - -void * -UFSStoreState::operator new (size_t) -{ - CBDATA_INIT_TYPE(UFSStoreState); - return cbdataAlloc(UFSStoreState); -} - -void -UFSStoreState::operator delete (void *address) -{ - cbdataFree(address); -} - -void -UFSStoreState::ioCompletedNotification() -{ - if (opening) { - opening = false; - debugs(79, 3, "UFSStoreState::ioCompletedNotification: dirno " << - swap_dirn << ", fileno "<< std::setfill('0') << std::hex << - std::setw(8) << swap_filen << " status "<< std::setfill(' ') << - std::dec << theFile->error()); - - assert (FILE_MODE(mode) == O_RDONLY); - openDone(); - - return; - } - - if (creating) { - creating = false; - debugs(79, 3, "UFSStoreState::ioCompletedNotification: dirno " << - swap_dirn << ", fileno "<< std::setfill('0') << std::hex << - std::setw(8) << swap_filen << " status "<< std::setfill(' ') << - std::dec << theFile->error()); - - openDone(); - - return; - } - - assert (!(closing ||opening)); - debugs(79, 3, "diskd::ioCompleted: dirno " << swap_dirn << ", fileno "<< - std::setfill('0') << std::hex << std::setw(8) << swap_filen << - " status "<< std::setfill(' ') << std::dec << theFile->error()); - - /* Ok, notification past open means an error has occured */ - assert (theFile->error()); - tryClosing(); -} - -void -UFSStoreState::openDone() -{ - if (closing) - debugs(0, DBG_CRITICAL, HERE << "already closing in openDone()!?"); - - if (theFile->error()) { - tryClosing(); - return; - } - - if (FILE_MODE(mode) == O_WRONLY) { - drainWriteQueue(); - - } else if ((FILE_MODE(mode) == O_RDONLY) && !closing) { - if (kickReadQueue()) - return; - } - - if (flags.try_closing) - tryClosing(); - - debugs(79, 3, "UFSStoreState::openDone: exiting"); -} - -void -UFSStoreState::closeCompleted() -{ - assert (closing); - debugs(79, 3, "UFSStoreState::closeCompleted: dirno " << swap_dirn << - ", fileno "<< std::setfill('0') << std::hex << std::setw(8) << - swap_filen << " status "<< std::setfill(' ') << std::dec << - theFile->error()); - - if (theFile->error()) { - debugs(79,3,HERE<< "theFile->error() ret " << theFile->error()); - doCloseCallback(DISK_ERROR); - } else { - doCloseCallback(DISK_OK); - } - - closing = false; -} - -/* - * DPW 2006-05-24 - * This close function is called by the higher layer when it has finished - * reading/writing everything, or otherwise wants to close the swap - * file. In the case of writing and using aufs storage, close() might - * be called before any/all data is written, and even before the open - * callback occurs. Thus, we use our tryClosing() method, which knows - * when it is safe to actually signal the lower layer for closing. - */ -void -UFSStoreState::close(int) -{ - debugs(79, 3, "UFSStoreState::close: dirno " << swap_dirn << ", fileno "<< - std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen); - tryClosing(); // UFS does not distinguish different closure types -} - -void -UFSStoreState::read_(char *buf, size_t size, off_t aOffset, STRCB * aCallback, void *aCallbackData) -{ - assert(read.callback == NULL); - assert(read.callback_data == NULL); - assert(!reading); - assert(!closing); - assert (aCallback); - - if (!theFile->canRead()) { - debugs(79, 3, "UFSStoreState::read_: queueing read because theFile can't read"); - queueRead (buf, size, aOffset, aCallback, aCallbackData); - return; - } - - read.callback = aCallback; - read.callback_data = cbdataReference(aCallbackData); - debugs(79, 3, "UFSStoreState::read_: dirno " << swap_dirn << ", fileno "<< - std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen); - offset_ = aOffset; - read_buf = buf; - reading = true; - theFile->read(new ReadRequest(buf,aOffset,size)); -} - - -/* - * DPW 2006-05-24 - * This, the public write interface, places the write request at the end - * of the pending_writes queue to ensure correct ordering of writes. - * We could optimize things a little if there are no other pending - * writes and just do the write directly. But for now we'll keep the - * code simpler and always go through the pending_writes queue. - */ -void -UFSStoreState::write(char const *buf, size_t size, off_t aOffset, FREE * free_func) -{ - debugs(79, 3, "UFSStoreState::write: dirn " << swap_dirn << ", fileno "<< - std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen); - - if (theFile->error()) { - debugs(79, DBG_IMPORTANT,HERE << "avoid write on theFile with error"); - debugs(79, DBG_IMPORTANT,HERE << "calling free_func for " << (void*) buf); - free_func((void*)buf); - return; - } - - queueWrite(buf, size, aOffset, free_func); - drainWriteQueue(); -} - - -/* - * DPW 2006-05-24 - * This, the private write method, calls the lower level write for the - * first write request in the pending_writes queue. doWrite() is only - * called by drainWriteQueue(). - */ -void -UFSStoreState::doWrite() -{ - debugs(79, 3, HERE << this << " UFSStoreState::doWrite"); - - assert(theFile->canWrite()); - - _queued_write *q = (_queued_write *)linklistShift(&pending_writes); - - if (q == NULL) { - debugs(79, 3, HERE << this << " UFSStoreState::doWrite queue is empty"); - return; - } - - if (theFile->error()) { - debugs(79, DBG_IMPORTANT,HERE << "avoid write on theFile with error"); - debugs(79,3,HERE << "calling free_func for " << (void*) q->buf); - /* - * DPW 2006-05-24 - * Note "free_func" is memNodeWriteComplete(), which doesn't - * really free the memory. Instead it clears the node's - * write_pending flag. - */ - q->free_func((void*)q->buf); - delete q; - return; - } - - /* - * DPW 2006-05-24 - * UFSStoreState has a 'writing' flag that we used to set here, - * but it wasn't really used anywhere. In fact, some lower - * layers such as DISKD allow multiple outstanding writes, which - * makes the boolean writing flag meaningless. We would need - * a counter to keep track of writes going out and write callbacks - * coming in. For now let's just not use the writing flag at - * all. - */ - debugs(79, 3, HERE << this << " calling theFile->write(" << q->size << ")"); - - theFile->write(new WriteRequest(q->buf, q->offset, q->size, q->free_func)); - delete q; -} - -void -UFSStoreState::readCompleted(const char *buf, int len, int errflag, RefCount result) -{ - assert (result.getRaw()); - reading = false; - debugs(79, 3, "UFSStoreState::readCompleted: dirno " << swap_dirn << - ", fileno "<< std::setfill('0') << std::hex << std::setw(8) << - swap_filen << " len "<< std::setfill(' ') << std::dec << len); - - if (len > 0) - offset_ += len; - - STRCB *callback_ = read.callback; - - assert(callback_); - - read.callback = NULL; - - void *cbdata; - - /* A note: - * diskd IO queues closes via the diskd queue. So close callbacks - * occur strictly after reads and writes. - * ufs doesn't queue, it simply completes, so close callbacks occur - * strictly after reads and writes. - * aufs performs closes syncronously, so close events must be managed - * to force strict ordering. - * The below does this: - * closing is set when theFile->close() has been called, and close only triggers - * when no io's are pending. - * writeCompleted likewise. - */ - if (!closing && cbdataReferenceValidDone(read.callback_data, &cbdata)) { - if (len > 0 && read_buf != buf) - memcpy(read_buf, buf, len); - - callback_(cbdata, read_buf, len, this); - } - - if (flags.try_closing || (theFile != NULL && theFile->error()) ) - tryClosing(); -} - -void -UFSStoreState::writeCompleted(int errflag, size_t len, RefCount writeRequest) -{ - debugs(79, 3, HERE << "dirno " << swap_dirn << ", fileno " << - std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen << - ", len " << len); - /* - * DPW 2006-05-24 - * See doWrites() for why we don't update UFSStoreState::writing - * here anymore. - */ - - offset_ += len; - - if (theFile->error()) { - debugs(79,2,HERE << " detected an error, will try to close"); - tryClosing(); - } - - /* - * HNO 2009-07-24 - * Kick any pending write/close operations alive - */ - drainWriteQueue(); -} - -void -UFSStoreState::doCloseCallback(int errflag) -{ - debugs(79, 3, "storeUfsIOCallback: errflag=" << errflag); - /* - * DPW 2006-05-24 - * When we signal the higher layer with this callback, it might unlock - * the StoreEntry and its associated data. We must "free" any queued - * I/Os (especially writes) now, otherwise the StoreEntry's mem_node's - * will have their write_pending flag set, and we'll get an assertion. - */ - freePending(); - STIOCB *theCallback = callback; - callback = NULL; - - void *cbdata; - - if (cbdataReferenceValidDone(callback_data, &cbdata) && theCallback) - theCallback(cbdata, errflag, this); - - /* - * We are finished with theFile since the lower layer signalled - * us that the file has been closed. This must be the last line, - * as theFile may be the only object holding us in memory. - */ - theFile = NULL; // refcounted -} - -/* ============= THE REAL UFS CODE ================ */ - -UFSStoreState::UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callback_, void *callback_data_) : opening (false), creating (false), closing (false), reading(false), writing(false), pending_reads(NULL), pending_writes (NULL) -{ - swap_filen = anEntry->swap_filen; - swap_dirn = SD->index; - mode = O_BINARY; - callback = callback_; - callback_data = cbdataReference(callback_data_); - e = anEntry; - flags.write_draining = false; - flags.try_closing = false; -} - -UFSStoreState::~UFSStoreState() -{ - assert(pending_reads == NULL); - assert(pending_writes == NULL); -} - -void -UFSStoreState::freePending() -{ - _queued_read *qr; - - while ((qr = (_queued_read *)linklistShift(&pending_reads))) { - cbdataReferenceDone(qr->callback_data); - delete qr; - } - - debugs(79,3,HERE << "UFSStoreState::freePending: freed pending reads"); - - _queued_write *qw; - - while ((qw = (_queued_write *)linklistShift(&pending_writes))) { - if (qw->free_func) - qw->free_func(const_cast(qw->buf)); - delete qw; - } - - debugs(79,3,HERE << "UFSStoreState::freePending: freed pending writes"); -} - -bool -UFSStoreState::kickReadQueue() -{ - _queued_read *q = (_queued_read *)linklistShift(&pending_reads); - - if (NULL == q) - return false; - - debugs(79, 3, "UFSStoreState::kickReadQueue: reading queued request of " << q->size << " bytes"); - - void *cbdata; - - if (cbdataReferenceValidDone(q->callback_data, &cbdata)) { - read_(q->buf, q->size, q->offset, q->callback, cbdata); - } else { - debugs(79, 2, "UFSStoreState::kickReadQueue: this: " << this << " cbdataReferenceValidDone returned false." << " closing: " << closing << " flags.try_closing: " << flags.try_closing); - delete q; - return false; - } - - delete q; - - return true; -} - -void -UFSStoreState::queueRead(char *buf, size_t size, off_t aOffset, STRCB *callback_, void *callback_data_) -{ - debugs(79, 3, "UFSStoreState::queueRead: queueing read"); - assert(opening); - assert (pending_reads == NULL); - _queued_read *q = new _queued_read; - q->buf = buf; - q->size = size; - q->offset = aOffset; - q->callback = callback_; - q->callback_data = cbdataReference(callback_data_); - linklistPush(&pending_reads, q); -} - -/* - * DPW 2006-05-24 - * drainWriteQueue() is a loop around doWrite(). - */ -void -UFSStoreState::drainWriteQueue() -{ - /* - * DPW 2007-04-12 - * We might find that flags.write_draining is already set - * because schemes like diskd can process I/O acks - * before sending another I/O request. e.g. the following - * sequence of events: open request -> write request -> - * drainWriteQueue() -> queue full -> callbacks -> openDone() -> - * drainWriteQueue(). - */ - if (flags.write_draining) - return; - - if (!theFile->canWrite()) - return; - - flags.write_draining = true; - - while (pending_writes != NULL) { - doWrite(); - } - - flags.write_draining = false; - - if (flags.try_closing) - tryClosing(); -} - -/* - * DPW 2006-05-24 - * This blows. DiskThreadsDiskFile::close() won't actually do the close - * if ioInProgress() is true. So we have to check it here. Maybe someday - * DiskThreadsDiskFile::close() will be modified to have a return value, - * or will remember to do the close for us. - */ -void -UFSStoreState::tryClosing() -{ - debugs(79,3,HERE << this << " tryClosing()" << - " closing = " << closing << - " flags.try_closing = " << flags.try_closing << - " ioInProgress = " << theFile->ioInProgress()); - - if (theFile->ioInProgress()) { - debugs(79, 3, HERE << this << - " won't close since ioInProgress is true, bailing"); - flags.try_closing = true; - return; - } - - closing = true; - flags.try_closing = false; - theFile->close(); -} - -void -UFSStoreState::queueWrite(char const *buf, size_t size, off_t aOffset, FREE * free_func) -{ - debugs(79, 3, HERE << this << " UFSStoreState::queueWrite: queueing write of size " << size); - - _queued_write *q; - q = new _queued_write; - q->buf = buf; - q->size = size; - q->offset = aOffset; - q->free_func = free_func; - linklistPush(&pending_writes, q); -} - -StoreIOState::Pointer -UFSStrategy::open(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback, - StoreIOState::STIOCB * aCallback, void *callback_data) -{ - assert (((UFSSwapDir *)SD)->IO == this); - debugs(79, 3, "UFSStrategy::open: fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e->swap_filen); - - /* to consider: make createstate a private UFSStrategy call */ - StoreIOState::Pointer sio = createState (SD, e, aCallback, callback_data); - - sio->mode |= O_RDONLY; - - UFSStoreState *state = dynamic_cast (sio.getRaw()); - - assert (state); - - char *path = ((UFSSwapDir *)SD)->fullPath(e->swap_filen, NULL); - - DiskFile::Pointer myFile = newFile (path); - - if (myFile.getRaw() == NULL) - return NULL; - - state->theFile = myFile; - - state->opening = true; - - myFile->open (sio->mode, 0644, state); - - if (myFile->error()) - return NULL; - - return sio; -} - -StoreIOState::Pointer -UFSStrategy::create(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback, - StoreIOState::STIOCB * aCallback, void *callback_data) -{ - assert (((UFSSwapDir *)SD)->IO == this); - /* Allocate a number */ - sfileno filn = ((UFSSwapDir *)SD)->mapBitAllocate(); - debugs(79, 3, "UFSStrategy::create: fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << filn); - - /* Shouldn't we handle a 'bitmap full' error here? */ - - StoreIOState::Pointer sio = createState (SD, e, aCallback, callback_data); - - sio->mode |= O_WRONLY | O_CREAT | O_TRUNC; - - sio->swap_filen = filn; - - UFSStoreState *state = dynamic_cast (sio.getRaw()); - - assert (state); - - char *path = ((UFSSwapDir *)SD)->fullPath(filn, NULL); - - DiskFile::Pointer myFile = newFile (path); - - if (myFile.getRaw() == NULL) { - ((UFSSwapDir *)SD)->mapBitReset (filn); - return NULL; - } - - state->theFile = myFile; - - state->creating = true; - - myFile->create (state->mode, 0644, state); - - if (myFile->error()) { - ((UFSSwapDir *)SD)->mapBitReset (filn); - return NULL; - } - - /* now insert into the replacement policy */ - ((UFSSwapDir *)SD)->replacementAdd(e); - - return sio; -} - -int -UFSStrategy::callback() -{ - return io->callback(); -} - -void -UFSStrategy::init() -{ - io->init(); -} - -void -UFSStrategy::sync() -{ - io->sync(); -} - -void -UFSStrategy::statfs(StoreEntry & sentry)const -{ - io->statfs(sentry); -} - === removed file 'src/fs/ufs/ufscommon.cc' --- src/fs/ufs/ufscommon.cc 2012-07-23 07:02:06 +0000 +++ src/fs/ufs/ufscommon.cc 1970-01-01 00:00:00 +0000 @@ -1,841 +0,0 @@ -/* - * $Id$ - * - * DEBUG: section 47 Store Directory Routines - * AUTHOR: Robert Collins - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - * Copyright (c) 2003, Robert Collins - */ - -#include "squid.h" -#include "ufscommon.h" -#include "Store.h" -#include "fde.h" -#include "SquidTime.h" -#include "StoreMeta.h" -#include "Generic.h" -#include "StoreMetaUnpacker.h" -#include "RefCount.h" -#include "StoreSwapLogData.h" -#include "swap_log_op.h" - - -CBDATA_CLASS_INIT(RebuildState); - -/// Parse a swap header entry created on a system with 32-bit size_t and sfileno -/// this is typical of 32-bit systems without large file support -/// NP: SQUID_MD5_DIGEST_LENGTH is very risky still. -class UFSSwapLogParser_v1_32bs:public UFSSwapLogParser -{ -public: - /// version 1 cache swap.state entry with 32-bit size_t (swap_file_sz) - /// time_t an sfileno have no variation from the v1 baseline format - struct StoreSwapLogDataOld { - char op; - sfileno swap_filen; - time_t timestamp; - time_t lastref; - time_t expires; - time_t lastmod; - uint32_t swap_file_sz; - uint16_t refcount; - uint16_t flags; - unsigned char key[SQUID_MD5_DIGEST_LENGTH]; - }; - UFSSwapLogParser_v1_32bs(FILE *fp):UFSSwapLogParser(fp) { - record_size = sizeof(UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld); - } - /// Convert the on-disk 32-bit format to our current format while reading - bool ReadRecord(StoreSwapLogData &swapData) { - UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld readData; - int bytes = sizeof(UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld); - - assert(log); - - if (fread(&readData, bytes, 1, log) != 1) { - return false; - } - swapData.op = readData.op; - swapData.swap_filen = readData.swap_filen; - swapData.timestamp = readData.timestamp; - swapData.lastref = readData.lastref; - swapData.expires = readData.expires; - swapData.lastmod = readData.lastmod; - swapData.swap_file_sz = readData.swap_file_sz; - swapData.refcount = readData.refcount; - swapData.flags = readData.flags; - memcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); - return true; - } -}; - -#if UNUSED_CODE -/// Parse a swap header entry created on a system with 32-bit size_t, time_t and sfileno -/// this is typical of 32-bit systems without large file support and with old kernels -/// NP: SQUID_MD5_DIGEST_LENGTH is very risky still. -class UFSSwapLogParser_v1_32bst:public UFSSwapLogParser -{ -public: - /// version 1 cache swap.state entry with 32-bit size_t (swap_file_sz) - /// time_t also differs - /// sfileno has no variation from the v1 baseline format - struct StoreSwapLogDataOld { - char op; - sfileno swap_filen; - int32_t timestamp; - int32_t lastref; - int32_t expires; - int32_t lastmod; - uint32_t swap_file_sz; - uint16_t refcount; - uint16_t flags; - unsigned char key[SQUID_MD5_DIGEST_LENGTH]; - }; - UFSSwapLogParser_v1_32bst(FILE *fp):UFSSwapLogParser(fp) { - record_size = sizeof(UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld); - } - /// Convert the on-disk 32-bit format to our current format while reading - bool ReadRecord(StoreSwapLogData &swapData) { - UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld readData; - int bytes = sizeof(UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld); - - assert(log); - - if (fread(&readData, bytes, 1, log) != 1) { - return false; - } - swapData.op = readData.op; - swapData.swap_filen = readData.swap_filen; - swapData.timestamp = readData.timestamp; - swapData.lastref = readData.lastref; - swapData.expires = readData.expires; - swapData.lastmod = readData.lastmod; - swapData.swap_file_sz = readData.swap_file_sz; - swapData.refcount = readData.refcount; - swapData.flags = readData.flags; - memcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); - return true; - } -}; - -/// Parse a swap header entry created on a system with 64-bit size_t and sfileno -/// this is typical of 64-bit systems prior to this patch fixing sfileno to 32-bits -/// NP: SQUID_MD5_DIGEST_LENGTH is very risky still. -class UFSSwapLogParser_v1_64bfn:public UFSSwapLogParser -{ -public: - /// version 1 cache swap.state entry with 64-bit sfileno - struct StoreSwapLogDataOld { - char op; - int64_t swap_filen; - time_t timestamp; - time_t lastref; - time_t expires; - time_t lastmod; - uint64_t swap_file_sz; - uint16_t refcount; - uint16_t flags; - unsigned char key[SQUID_MD5_DIGEST_LENGTH]; - }; - UFSSwapLogParser_v1_64bfn(FILE *fp):UFSSwapLogParser(fp) { - record_size = sizeof(UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld); - } - /// Convert the on-disk 64-bit format to our current format while reading - bool ReadRecord(StoreSwapLogData &swapData) { - UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld readData; - int bytes = sizeof(UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld); - - assert(log); - - if (fread(&readData, bytes, 1, log) != 1) { - return false; - } - swapData.op = readData.op; - if ((readData.swap_filen>>32) != 0) { - fatalf("File ID on record is greater than maximum cache file ID."); - } - swapData.swap_filen = (int32_t)readData.swap_filen; - swapData.timestamp = readData.timestamp; - swapData.lastref = readData.lastref; - swapData.expires = readData.expires; - swapData.lastmod = readData.lastmod; - swapData.swap_file_sz = readData.swap_file_sz; - swapData.refcount = readData.refcount; - swapData.flags = readData.flags; - memcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); - return true; - } -}; - -class UFSSwapLogParser_v1:public UFSSwapLogParser -{ -public: - UFSSwapLogParser_v1(FILE *fp):UFSSwapLogParser(fp) { - record_size = sizeof(StoreSwapLogData); - } - bool ReadRecord(StoreSwapLogData &swapData); -}; - - -bool UFSSwapLogParser_v1::ReadRecord(StoreSwapLogData &swapData) -{ - int bytes = sizeof(StoreSwapLogData); - - assert(log); - - if (fread(&swapData, bytes, 1, log) != 1) { - return false; - } - return true; -} -#endif /* UNUSED_CODE */ - -/// swap.state v2 log parser -class UFSSwapLogParser_v2: public UFSSwapLogParser -{ -public: - UFSSwapLogParser_v2(FILE *fp): UFSSwapLogParser(fp) { - record_size = sizeof(StoreSwapLogData); - } - bool ReadRecord(StoreSwapLogData &swapData) { - assert(log); - return fread(&swapData, sizeof(StoreSwapLogData), 1, log) == 1; - } -}; - - -UFSSwapLogParser *UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp) -{ - StoreSwapLogHeader header; - - assert(fp); - - if (fread(&header, sizeof(StoreSwapLogHeader), 1, fp) != 1) - return NULL; - - if (header.op != SWAP_LOG_VERSION) { - debugs(47, 1, "Old swap file detected..."); - fseek(fp, 0, SEEK_SET); - return new UFSSwapLogParser_v1_32bs(fp); // Um. 32-bits except time_t, and can't determine that. - } - - debugs(47, 2, "Swap file version: " << header.version); - - if (header.version == 1) { - if (fseek(fp, header.record_size, SEEK_SET) != 0) - return NULL; - - debugs(47, DBG_IMPORTANT, "Rejecting swap file v1 to avoid cache " << - "index corruption. Forcing a full cache index rebuild. " << - "See Squid bug #3441."); - return NULL; - -#if UNUSED_CODE - // baseline - // 32-bit sfileno - // native time_t (hopefully 64-bit) - // 64-bit file size - if (header.record_size == sizeof(StoreSwapLogData)) { - debugs(47, 1, "Version 1 of swap file with LFS support detected... "); - return new UFSSwapLogParser_v1(fp); - } - - // which means we have a 3-way grid of permutations to import (yuck!) - // 1) sfileno 32-bit / 64-bit (64-bit was broken) - // 2) time_t 32-bit / 64-bit - // 3) size_t 32-bit / 64-bit (32-bit was pre-LFS) - - // 32-bit systems... - // only LFS (size_t) differs from baseline - if (header.record_size == sizeof(struct UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld)) { - debugs(47, 1, "Version 1 (32-bit) swap file without LFS support detected... "); - return new UFSSwapLogParser_v1_32bs(fp); - } - // LFS (size_t) and timestamps (time_t) differs from baseline - if (header.record_size == sizeof(struct UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld)) { - debugs(47, 1, "Version 1 (32-bit) swap file with short timestamps and without LFS support detected... "); - return new UFSSwapLogParser_v1_32bst(fp); - } - // No downgrade for 64-bit timestamps to 32-bit. - - // 64-bit systems - // sfileno was 64-bit for a some builds - if (header.record_size == sizeof(struct UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld)) { - debugs(47, 1, "Version 1 (64-bit) swap file with broken sfileno detected... "); - return new UFSSwapLogParser_v1_64bfn(fp); - } - // NP: 64-bit system with 32-bit size_t/time_t are not handled. - - debugs(47, 1, "WARNING: The swap file has wrong format!... "); - debugs(47, 1, "NOTE: Cannot safely downgrade caches to short (32-bit) timestamps."); - return NULL; -#endif - } - - if (header.version >= 2) { - if (!header.sane()) { - debugs(47, DBG_IMPORTANT, "ERROR: Corrupted v" << header.version << - " swap file header."); - return NULL; - } - - if (fseek(fp, header.record_size, SEEK_SET) != 0) - return NULL; - - if (header.version == 2) - return new UFSSwapLogParser_v2(fp); - } - - // TODO: v3: write to disk in network-order bytes for the larger fields? - - debugs(47, DBG_IMPORTANT, "Unknown swap file version: " << header.version); - return NULL; -} - -int UFSSwapLogParser::SwapLogEntries() -{ - struct stat sb; - - if (log_entries >= 0) - return log_entries; - - if (log && record_size && 0 == fstat(fileno(log), &sb)) { - log_entries = sb.st_size/record_size; - return log_entries; - } - - return 0; -} - - - - -RebuildState::RebuildState (RefCount aSwapDir) : sd (aSwapDir),LogParser(NULL), e(NULL), fromLog(true), _done (false) -{ - /* - * If the swap.state file exists in the cache_dir, then - * we'll use commonUfsDirRebuildFromSwapLog(), otherwise we'll - * use commonUfsDirRebuildFromDirectory() to open up each file - * and suck in the meta data. - */ - int clean = 0; - int zeroLengthLog = 0; - FILE *fp = sd->openTmpSwapLog(&clean, &zeroLengthLog); - - if (fp && !zeroLengthLog) - LogParser = UFSSwapLogParser::GetUFSSwapLogParser(fp); - - if (LogParser == NULL ) { - fromLog = false; - - if (fp != NULL) - fclose(fp); - - } else { - fromLog = true; - flags.clean = (unsigned int) clean; - } - - if (!clean) - flags.need_to_validate = 1; - - debugs(47, DBG_IMPORTANT, "Rebuilding storage in " << sd->path << " (" << - (clean ? "clean log" : (LogParser ? "dirty log" : "no log")) << ")"); -} - -RebuildState::~RebuildState() -{ - sd->closeTmpSwapLog(); - - if (LogParser) - delete LogParser; -} - -void -RebuildState::RebuildStep(void *data) -{ - RebuildState *rb = (RebuildState *)data; - rb->rebuildStep(); - - if (!rb->isDone()) - eventAdd("storeRebuild", RebuildStep, rb, 0.01, 1); - else { - -- StoreController::store_dirs_rebuilding; - storeRebuildComplete(&rb->counts); - delete rb; - } -} - -/// load entries from swap.state or files until we run out of entries or time -void -RebuildState::rebuildStep() -{ - currentEntry(NULL); - - // Balance our desire to maximize the number of entries processed at once - // (and, hence, minimize overheads and total rebuild time) with a - // requirement to also process Coordinator events, disk I/Os, etc. - const int maxSpentMsec = 50; // keep small: most RAM I/Os are under 1ms - const timeval loopStart = current_time; - - const int totalEntries = LogParser ? LogParser->SwapLogEntries() : -1; - - while (!isDone()) { - if (fromLog) - rebuildFromSwapLog(); - else - rebuildFromDirectory(); - - // TODO: teach storeRebuildProgress to handle totalEntries <= 0 - if (totalEntries > 0 && (n_read % 4000 == 0)) - storeRebuildProgress(sd->index, totalEntries, n_read); - - if (opt_foreground_rebuild) - continue; // skip "few entries at a time" check below - - getCurrentTime(); - const double elapsedMsec = tvSubMsec(loopStart, current_time); - if (elapsedMsec > maxSpentMsec || elapsedMsec < 0) { - debugs(47, 5, HERE << "pausing after " << n_read << " entries in " << - elapsedMsec << "ms; " << (elapsedMsec/n_read) << "ms per entry"); - break; - } - } -} - -/// process one cache file -void -RebuildState::rebuildFromDirectory() -{ - cache_key key[SQUID_MD5_DIGEST_LENGTH]; - - struct stat sb; - int fd = -1; - assert(this != NULL); - debugs(47, 3, "commonUfsDirRebuildFromDirectory: DIR #" << sd->index); - - assert(fd == -1); - sfileno filn = 0; - int size; - fd = getNextFile(&filn, &size); - - if (fd == -2) { - debugs(47, DBG_IMPORTANT, "Done scanning " << sd->path << " dir (" << - n_read << " entries)"); - _done = true; - return; - } else if (fd < 0) { - return; - } - - assert(fd > -1); - /* lets get file stats here */ - - ++n_read; - - if (fstat(fd, &sb) < 0) { - debugs(47, 1, "commonUfsDirRebuildFromDirectory: fstat(FD " << fd << "): " << xstrerror()); - file_close(fd); - --store_open_disk_fd; - fd = -1; - return; - } - - MemBuf buf; - buf.init(SM_PAGE_SIZE, SM_PAGE_SIZE); - if (!storeRebuildLoadEntry(fd, sd->index, buf, counts)) - return; - - StoreEntry tmpe; - const bool loaded = storeRebuildParseEntry(buf, tmpe, key, counts, - (int64_t)sb.st_size); - - file_close(fd); - --store_open_disk_fd; - fd = -1; - - if (!loaded) { - // XXX: shouldn't this be a call to commonUfsUnlink? - sd->unlinkFile(filn); // should we unlink in all failure cases? - return; - } - - if (!storeRebuildKeepEntry(tmpe, key, counts)) - return; - - ++counts.objcount; - // tmpe.dump(5); - currentEntry(sd->addDiskRestore(key, - filn, - tmpe.swap_file_sz, - tmpe.expires, - tmpe.timestamp, - tmpe.lastref, - tmpe.lastmod, - tmpe.refcount, /* refcount */ - tmpe.flags, /* flags */ - (int) flags.clean)); - storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); -} - -StoreEntry * -RebuildState::currentEntry() const -{ - return e; -} - -void -RebuildState::currentEntry(StoreEntry *newValue) -{ - e = newValue; -} - -/// process one swap log entry -void -RebuildState::rebuildFromSwapLog() -{ - StoreSwapLogData swapData; - - if (LogParser->ReadRecord(swapData) != 1) { - debugs(47, 1, "Done reading " << sd->path << " swaplog (" << n_read << " entries)"); - LogParser->Close(); - delete LogParser; - LogParser = NULL; - _done = true; - return; - } - - ++n_read; - - if (!swapData.sane()) { - ++counts.invalid; - return; - } - - /* - * BC: during 2.4 development, we changed the way swap file - * numbers are assigned and stored. The high 16 bits used - * to encode the SD index number. There used to be a call - * to storeDirProperFileno here that re-assigned the index - * bits. Now, for backwards compatibility, we just need - * to mask it off. - */ - swapData.swap_filen &= 0x00FFFFFF; - - debugs(47, 3, "commonUfsDirRebuildFromSwapLog: " << - swap_log_op_str[(int) swapData.op] << " " << - storeKeyText(swapData.key) << " "<< std::setfill('0') << - std::hex << std::uppercase << std::setw(8) << - swapData.swap_filen); - - if (swapData.op == SWAP_LOG_ADD) { - (void) 0; - } else if (swapData.op == SWAP_LOG_DEL) { - /* Delete unless we already have a newer copy anywhere in any store */ - /* this needs to become - * 1) unpack url - * 2) make synthetic request with headers ?? or otherwise search - * for a matching object in the store - * TODO FIXME change to new async api - */ - currentEntry (Store::Root().get(swapData.key)); - - if (currentEntry() != NULL && swapData.lastref >= e->lastref) { - undoAdd(); - --counts.objcount; - ++counts.cancelcount; - } - return; - } else { - const double - x = ::log(static_cast(++counts.bad_log_op)) / ::log(10.0); - - if (0.0 == x - (double) (int) x) - debugs(47, 1, "WARNING: " << counts.bad_log_op << " invalid swap log entries found"); - - ++counts.invalid; - - return; - } - - ++counts.scancount; // XXX: should not this be incremented earlier? - - if (!sd->validFileno(swapData.swap_filen, 0)) { - ++counts.invalid; - return; - } - - if (EBIT_TEST(swapData.flags, KEY_PRIVATE)) { - ++counts.badflags; - return; - } - - /* this needs to become - * 1) unpack url - * 2) make synthetic request with headers ?? or otherwise search - * for a matching object in the store - * TODO FIXME change to new async api - */ - currentEntry (Store::Root().get(swapData.key)); - - int used; /* is swapfile already in use? */ - - used = sd->mapBitTest(swapData.swap_filen); - - /* If this URL already exists in the cache, does the swap log - * appear to have a newer entry? Compare 'lastref' from the - * swap log to e->lastref. */ - /* is the log entry newer than current entry? */ - int disk_entry_newer = currentEntry() ? (swapData.lastref > currentEntry()->lastref ? 1 : 0) : 0; - - if (used && !disk_entry_newer) { - /* log entry is old, ignore it */ - ++counts.clashcount; - return; - } else if (used && currentEntry() && currentEntry()->swap_filen == swapData.swap_filen && currentEntry()->swap_dirn == sd->index) { - /* swapfile taken, same URL, newer, update meta */ - - if (currentEntry()->store_status == STORE_OK) { - currentEntry()->lastref = swapData.timestamp; - currentEntry()->timestamp = swapData.timestamp; - currentEntry()->expires = swapData.expires; - currentEntry()->lastmod = swapData.lastmod; - currentEntry()->flags = swapData.flags; - currentEntry()->refcount += swapData.refcount; - sd->dereference(*currentEntry()); - } else { - debug_trap("commonUfsDirRebuildFromSwapLog: bad condition"); - debugs(47, 1, "\tSee " << __FILE__ << ":" << __LINE__); - } - return; - } else if (used) { - /* swapfile in use, not by this URL, log entry is newer */ - /* This is sorta bad: the log entry should NOT be newer at this - * point. If the log is dirty, the filesize check should have - * caught this. If the log is clean, there should never be a - * newer entry. */ - debugs(47, 1, "WARNING: newer swaplog entry for dirno " << - sd->index << ", fileno "<< std::setfill('0') << std::hex << - std::uppercase << std::setw(8) << swapData.swap_filen); - - /* I'm tempted to remove the swapfile here just to be safe, - * but there is a bad race condition in the NOVM version if - * the swapfile has recently been opened for writing, but - * not yet opened for reading. Because we can't map - * swapfiles back to StoreEntrys, we don't know the state - * of the entry using that file. */ - /* We'll assume the existing entry is valid, probably because - * were in a slow rebuild and the the swap file number got taken - * and the validation procedure hasn't run. */ - assert(flags.need_to_validate); - ++counts.clashcount; - return; - } else if (currentEntry() && !disk_entry_newer) { - /* key already exists, current entry is newer */ - /* keep old, ignore new */ - ++counts.dupcount; - return; - } else if (currentEntry()) { - /* key already exists, this swapfile not being used */ - /* junk old, load new */ - undoAdd(); - --counts.objcount; - ++counts.dupcount; - } else { - /* URL doesnt exist, swapfile not in use */ - /* load new */ - (void) 0; - } - - ++counts.objcount; - - currentEntry(sd->addDiskRestore(swapData.key, - swapData.swap_filen, - swapData.swap_file_sz, - swapData.expires, - swapData.timestamp, - swapData.lastref, - swapData.lastmod, - swapData.refcount, - swapData.flags, - (int) flags.clean)); - - storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); -} - -/// undo the effects of adding an entry in rebuildFromSwapLog() -void -RebuildState::undoAdd() -{ - StoreEntry *added = currentEntry(); - assert(added); - currentEntry(NULL); - - // TODO: Why bother with these two if we are going to release?! - added->expireNow(); - added->releaseRequest(); - - if (added->swap_filen > -1) { - UFSSwapDir *sde = dynamic_cast(INDEXSD(added->swap_dirn)); - assert(sde); - sde->undoAddDiskRestore(added); - } - - added->release(); -} - -int -RebuildState::getNextFile(sfileno * filn_p, int *size) -{ - int fd = -1; - int dirs_opened = 0; - debugs(47, 3, "commonUfsDirGetNextFile: flag=" << flags.init << ", " << - sd->index << ": /"<< std::setfill('0') << std::hex << - std::uppercase << std::setw(2) << curlvl1 << "/" << std::setw(2) << - curlvl2); - - if (done) - return -2; - - while (fd < 0 && done == 0) { - fd = -1; - - if (0 == flags.init) { /* initialize, open first file */ - done = 0; - curlvl1 = 0; - curlvl2 = 0; - in_dir = 0; - flags.init = 1; - assert(Config.cacheSwap.n_configured > 0); - } - - if (0 == in_dir) { /* we need to read in a new directory */ - snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X", - sd->path, - curlvl1, curlvl2); - - if (dirs_opened) - return -1; - - td = opendir(fullpath); - - ++dirs_opened; - - if (td == NULL) { - debugs(47, 1, "commonUfsDirGetNextFile: opendir: " << fullpath << ": " << xstrerror()); - } else { - entry = readdir(td); /* skip . and .. */ - entry = readdir(td); - - if (entry == NULL && errno == ENOENT) - debugs(47, 1, "commonUfsDirGetNextFile: directory does not exist!."); - debugs(47, 3, "commonUfsDirGetNextFile: Directory " << fullpath); - } - } - - if (td != NULL && (entry = readdir(td)) != NULL) { - ++in_dir; - - if (sscanf(entry->d_name, "%x", &fn) != 1) { - debugs(47, 3, "commonUfsDirGetNextFile: invalid " << entry->d_name); - continue; - } - - if (!UFSSwapDir::FilenoBelongsHere(fn, sd->index, curlvl1, curlvl2)) { - debugs(47, 3, "commonUfsDirGetNextFile: "<< std::setfill('0') << - std::hex << std::uppercase << std::setw(8) << fn << - " does not belong in " << std::dec << sd->index << "/" << - curlvl1 << "/" << curlvl2); - - continue; - } - - if (sd->mapBitTest(fn)) { - debugs(47, 3, "commonUfsDirGetNextFile: Locked, continuing with next."); - continue; - } - - snprintf(fullfilename, MAXPATHLEN, "%s/%s", - fullpath, entry->d_name); - debugs(47, 3, "commonUfsDirGetNextFile: Opening " << fullfilename); - fd = file_open(fullfilename, O_RDONLY | O_BINARY); - - if (fd < 0) - debugs(47, 1, "commonUfsDirGetNextFile: " << fullfilename << ": " << xstrerror()); - else - ++store_open_disk_fd; - - continue; - } - - if (td != NULL) - closedir(td); - - td = NULL; - - in_dir = 0; - - if (sd->validL2(++curlvl2)) - continue; - - curlvl2 = 0; - - if (sd->validL1(++curlvl1)) - continue; - - curlvl1 = 0; - - done = 1; - } - - *filn_p = fn; - return fd; -} - -bool -RebuildState::error() const -{ - return false; -} - -bool -RebuildState::isDone() const -{ - return _done; -} - -StoreEntry * -RebuildState::currentItem() -{ - return currentEntry(); -} - -#if !_USE_INLINE_ -#include "ufscommon.cci" -#endif === removed file 'src/fs/ufs/ufscommon.cci' --- src/fs/ufs/ufscommon.cci 2009-01-21 03:47:47 +0000 +++ src/fs/ufs/ufscommon.cci 1970-01-01 00:00:00 +0000 @@ -1,34 +0,0 @@ -/* - * $Id$ - * - * DEBUG: section 47 Store Directory Routines - * AUTHOR: Duane Wessels - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - === removed file 'src/fs/ufs/ufscommon.h' --- src/fs/ufs/ufscommon.h 2012-08-07 15:12:08 +0000 +++ src/fs/ufs/ufscommon.h 1970-01-01 00:00:00 +0000 @@ -1,428 +0,0 @@ -/* - * $Id$ - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ -#ifndef SQUID_UFSCOMMON_H -#define SQUID_UFSCOMMON_H - - -#define DefaultLevelOneDirs 16 -#define DefaultLevelTwoDirs 256 -#define STORE_META_BUFSZ 4096 - -class UFSStrategy; -class ConfigOptionVector; -class DiskIOModule; -class StoreSearch; -class FileMap; - -#include "SwapDir.h" - -/// \ingroup UFS -class UFSSwapDir : public SwapDir -{ - -public: - static int IsUFSDir(SwapDir* sd); - static int DirClean(int swap_index); - static int FilenoBelongsHere(int fn, int F0, int F1, int F2); - - UFSSwapDir(char const *aType, const char *aModuleType); - virtual void init(); - virtual void create(); - virtual void dump(StoreEntry &) const; - ~UFSSwapDir(); - virtual StoreSearch *search(String const url, HttpRequest *); - virtual bool doubleCheck(StoreEntry &); - virtual bool unlinkdUseful() const; - virtual void unlink(StoreEntry &); - virtual void statfs(StoreEntry &)const; - virtual void maintain(); - virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const; - virtual void reference(StoreEntry &); - virtual bool dereference(StoreEntry &); - virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); - virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); - virtual void openLog(); - virtual void closeLog(); - virtual int writeCleanStart(); - virtual void writeCleanDone(); - virtual void logEntry(const StoreEntry & e, int op) const; - virtual void parse(int index, char *path); - virtual void reconfigure(); - virtual int callback(); - virtual void sync(); - virtual void swappedOut(const StoreEntry &e); - virtual uint64_t currentSize() const { return cur_size; } - virtual uint64_t currentCount() const { return n_disk_objects; } - - void unlinkFile(sfileno f); - // move down when unlink is a virtual method - //protected: - UFSStrategy *IO; - char *fullPath(sfileno, char *) const; - /* temp */ - void closeTmpSwapLog(); - FILE *openTmpSwapLog(int *clean_flag, int *zero_flag); - char *swapSubDir(int subdirn) const; - int mapBitTest(sfileno filn); - void mapBitReset(sfileno filn); - void mapBitSet(sfileno filn); - StoreEntry *addDiskRestore(const cache_key * key, - sfileno file_number, - uint64_t swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - uint32_t refcount, - uint16_t flags, - int clean); - /// Undo the effects of UFSSwapDir::addDiskRestore(). - void undoAddDiskRestore(StoreEntry *e); - int validFileno(sfileno filn, int flag) const; - int mapBitAllocate(); - virtual ConfigOption *getOptionTree() const; - - void *fsdata; - - bool validL2(int) const; - bool validL1(int) const; - - void replacementAdd(StoreEntry *e); - void replacementRemove(StoreEntry *e); - -protected: - FileMap *map; - int suggest; - int l1; - int l2; - -private: - void parseSizeL1L2(); - static int NumberOfUFSDirs; - static int * UFSDirToGlobalDirMapping; - bool pathIsDirectory(const char *path)const; - int swaplog_fd; - static EVH CleanEvent; - bool verifyCacheDirs(); - void rebuild(); - int createDirectory(const char *path, int); - void createSwapSubDirs(); - void dumpEntry(StoreEntry &) const; - char *logFile(char const *ext = NULL)const; - void changeIO(DiskIOModule *); - bool optionIOParse(char const *option, const char *value, int reconfiguring); - void optionIODump(StoreEntry * e) const; - mutable ConfigOptionVector *currentIOOptions; - char const *ioType; - uint64_t cur_size; ///< currently used space in the storage area - uint64_t n_disk_objects; ///< total number of objects stored -}; - -#include "RefCount.h" -#include "DiskIO/IORequestor.h" - -/** - * UFS dir specific IO calls - * - \todo This should be whittled away. - * DiskIOModule should be providing the entire needed API. - */ - -class DiskIOStrategy; - -class DiskFile; - -/// \ingroup UFS -class UFSStrategy -{ - -public: - UFSStrategy (DiskIOStrategy *); - virtual ~UFSStrategy (); - /* Not implemented */ - UFSStrategy (UFSStrategy const &); - UFSStrategy &operator=(UFSStrategy const &); - - virtual bool shedLoad(); - - virtual int load(); - - StoreIOState::Pointer createState(SwapDir *SD, StoreEntry *e, StoreIOState::STIOCB * callback, void *callback_data) const; - /* UFS specific */ - virtual RefCount newFile (char const *path); - StoreIOState::Pointer open(SwapDir *, StoreEntry *, StoreIOState::STFNCB *, - StoreIOState::STIOCB *, void *); - StoreIOState::Pointer create(SwapDir *, StoreEntry *, StoreIOState::STFNCB *, - StoreIOState::STIOCB *, void *); - - virtual void unlinkFile (char const *); - virtual void sync(); - - virtual int callback(); - - /** Init per-instance logic */ - virtual void init(); - - /** cachemgr output on the IO instance stats */ - virtual void statfs(StoreEntry & sentry)const; - - /** The io strategy in use */ - DiskIOStrategy *io; -protected: - - friend class UFSSwapDir; -}; - -/** Common ufs-store-dir logic */ - -class ReadRequest; - -/// \ingroup UFS -class UFSStoreState : public StoreIOState, public IORequestor -{ - -public: - void * operator new (size_t); - void operator delete (void *); - UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callback_, void *callback_data_); - ~UFSStoreState(); - virtual void close(int how); - virtual void closeCompleted(); - // protected: - virtual void ioCompletedNotification(); - virtual void readCompleted(const char *buf, int len, int errflag, RefCount); - virtual void writeCompleted(int errflag, size_t len, RefCount); - RefCount theFile; - bool opening; - bool creating; - bool closing; - bool reading; - bool writing; - void read_(char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data); - void write(char const *buf, size_t size, off_t offset, FREE * free_func); - -protected: - virtual void doCloseCallback (int errflag); - - class _queued_read - { - - public: - MEMPROXY_CLASS(UFSStoreState::_queued_read); - char *buf; - size_t size; - off_t offset; - STRCB *callback; - void *callback_data; - - }; - - class _queued_write - { - - public: - MEMPROXY_CLASS(UFSStoreState::_queued_write); - char const *buf; - size_t size; - off_t offset; - FREE *free_func; - - }; - - /** \todo These should be in the IO strategy */ - - struct { - /** - * DPW 2006-05-24 - * the write_draining flag is used to avoid recursion inside - * the UFSStoreState::drainWriteQueue() method. - */ - bool write_draining; - /** - * DPW 2006-05-24 - * The try_closing flag is set by UFSStoreState::tryClosing() - * when UFSStoreState wants to close the file, but cannot - * because of pending I/Os. If set, UFSStoreState will - * try to close again in the I/O callbacks. - */ - bool try_closing; - } flags; - link_list *pending_reads; - link_list *pending_writes; - void queueRead(char *, size_t, off_t, STRCB *, void *); - void queueWrite(char const *, size_t, off_t, FREE *); - bool kickReadQueue(); - void drainWriteQueue(); - void tryClosing(); - char *read_buf; - -private: - void openDone(); - void freePending(); - void doWrite(); - - CBDATA_CLASS(UFSStoreState); -}; - -MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_read); -MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_write); - - -#include "StoreSearch.h" - -/// \ingroup UFS -class StoreSearchUFS : public StoreSearch -{ - -public: - StoreSearchUFS(RefCount sd); - StoreSearchUFS(StoreSearchUFS const &); - virtual ~StoreSearchUFS(); - - /** \todo Iterator API - garh, wrong place */ - /** - * callback the client when a new StoreEntry is available - * or an error occurs - */ - virtual void next(void (callback)(void *cbdata), void *cbdata); - - /** - \retval true if a new StoreEntry is immediately available - \retval false if a new StoreEntry is NOT immediately available - */ - virtual bool next(); - - virtual bool error() const; - virtual bool isDone() const; - virtual StoreEntry *currentItem(); - - RefCount sd; - RemovalPolicyWalker *walker; - -private: - /// \bug (callback) should be hidden behind a proper human readable name - void (callback)(void *cbdata); - void *cbdata; - StoreEntry * current; - bool _done; - - CBDATA_CLASS2(StoreSearchUFS); -}; - - -class StoreSwapLogData; - -/// \ingroup UFS -class UFSSwapLogParser -{ - -public: - FILE *log; - int log_entries; - int record_size; - - UFSSwapLogParser(FILE *fp):log(fp),log_entries(-1), record_size(0) { - } - virtual ~UFSSwapLogParser() {}; - - static UFSSwapLogParser *GetUFSSwapLogParser(FILE *fp); - - virtual bool ReadRecord(StoreSwapLogData &swapData) = 0; - int SwapLogEntries(); - void Close() { - if (log) { - fclose(log); - log = NULL; - } - } -}; - - -/// \ingroup UFS -class RebuildState : public RefCountable -{ - -public: - static EVH RebuildStep; - - RebuildState(RefCount sd); - ~RebuildState(); - - virtual bool error() const; - virtual bool isDone() const; - virtual StoreEntry *currentItem(); - - RefCount sd; - int n_read; - /* FILE *log;*/ - UFSSwapLogParser *LogParser; - int curlvl1; - int curlvl2; - - struct { - unsigned int need_to_validate:1; - unsigned int clean:1; - unsigned int init:1; - } flags; - int in_dir; - int done; - int fn; - - dirent_t *entry; - DIR *td; - char fullpath[MAXPATHLEN]; - char fullfilename[MAXPATHLEN]; - - struct _store_rebuild_data counts; - -private: - void rebuildFromDirectory(); - void rebuildFromSwapLog(); - void rebuildStep(); - void undoAdd(); - int getNextFile(sfileno *, int *size); - StoreEntry *currentEntry() const; - void currentEntry(StoreEntry *); - StoreEntry *e; - bool fromLog; - bool _done; - /// \bug (callback) should be hidden behind a proper human readable name - void (callback)(void *cbdata); - void *cbdata; - - CBDATA_CLASS2(RebuildState); -}; - -#if _USE_INLINE_ -#include "ufscommon.cci" -#endif - -#endif /* SQUID_UFSCOMMON_H */ === modified file 'src/tests/testCoss.cc' --- src/tests/testCoss.cc 2012-01-20 18:55:04 +0000 +++ src/tests/testCoss.cc 2012-08-10 06:56:49 +0000 @@ -4,7 +4,6 @@ #include "Store.h" #include "SwapDir.h" #include "DiskIO/DiskIOModule.h" -#include "fs/ufs/ufscommon.h" #include "fs/coss/CossSwapDir.h" #include "Mem.h" #include "MemObject.h" === modified file 'src/tests/testDiskIO.cc' --- src/tests/testDiskIO.cc 2012-01-20 18:55:04 +0000 +++ src/tests/testDiskIO.cc 2012-08-10 06:56:49 +0000 @@ -5,7 +5,6 @@ #include "Store.h" #include "SwapDir.h" #include "DiskIO/DiskIOModule.h" -#include "fs/ufs/ufscommon.h" #if 0 // AYJ: COSS in Squid-3 is disabled. #include "fs/coss/CossSwapDir.h" #endif === modified file 'src/tests/testNull.cc' --- src/tests/testNull.cc 2012-01-20 18:55:04 +0000 +++ src/tests/testNull.cc 2012-08-10 06:56:49 +0000 @@ -4,7 +4,6 @@ #include "Store.h" #include "SwapDir.h" #include "DiskIO/DiskIOModule.h" -#include "fs/ufs/ufscommon.h" #include "fs/null/store_null.h" #include "Mem.h" #include "MemObject.h" === modified file 'src/tests/testUfs.cc' --- src/tests/testUfs.cc 2012-01-20 18:55:04 +0000 +++ src/tests/testUfs.cc 2012-08-10 06:56:49 +0000 @@ -1,15 +1,16 @@ #define SQUID_UNIT_TEST 1 #include "squid.h" -#include "testUfs.h" -#include "Store.h" -#include "SwapDir.h" + #include "DiskIO/DiskIOModule.h" -#include "fs/ufs/ufscommon.h" +#include "HttpHeader.h" +#include "HttpReply.h" #include "Mem.h" #include "MemObject.h" -#include "HttpHeader.h" -#include "HttpReply.h" #include "testStoreSupport.h" +#include "testUfs.h" +#include "Store.h" +#include "SwapDir.h" +#include "fs/ufs/UFSSwapDir.h" #if HAVE_STDEXCEPT #include @@ -19,7 +20,7 @@ CPPUNIT_TEST_SUITE_REGISTRATION( testUfs ); -typedef RefCount SwapDirPointer; +typedef RefCount SwapDirPointer; extern REMOVALPOLICYCREATE createRemovalPolicy_lru; /* XXX fails with --enable-removal-policies=heap */ static void @@ -89,9 +90,9 @@ Store::Root(new StoreController); - SwapDirPointer aStore (new UFSSwapDir("ufs", "Blocking")); + SwapDirPointer aStore (new Fs::Ufs::UFSSwapDir("ufs", "Blocking")); - aStore->IO = new UFSStrategy(DiskIOModule::Find("Blocking")->createStrategy()); + aStore->IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find("Blocking")->createStrategy()); addSwapDir(aStore); @@ -232,7 +233,7 @@ CPPUNIT_ASSERT(!store_table); // or StoreHashIndex ctor will abort below Store::Root(new StoreController); - SwapDirPointer aStore (new UFSSwapDir("ufs", "Blocking")); + SwapDirPointer aStore (new Fs::Ufs::UFSSwapDir("ufs", "Blocking")); addSwapDir(aStore); commonInit(); Config.replPolicy = new RemovalPolicySettings;