------------------------------------------------------------ revno: 13224 revision-id: chtsanti@users.sourceforge.net-20140112171545-kqu75kfey4wbygop parent: kinkie@squid-cache.org-20140110190104-97hv3sj5xm3qvu9g committer: Christos Tsantilas branch nick: trunk timestamp: Sun 2014-01-12 19:15:45 +0200 message: SMP shared cache with timeouts for squid This patch investigates the Ipc::MemMap class which is a shared cache with timeouts for use with squid SMP. TODO: Ipc::MemMap class has similar interfaces and functionality with the Ipc::StoreMap class. These two classes should implemented as kid classes of an Ipc::SharedCache class which encompass features from Ipc::MemMap and Ipc::StoreMap classes. This is a Measurement Factory project ------------------------------------------------------------ # Bazaar merge directive format 2 (Bazaar 0.90) # revision_id: chtsanti@users.sourceforge.net-20140112171545-\ # kqu75kfey4wbygop # target_branch: http://bzr.squid-cache.org/bzr/squid3/trunk/ # testament_sha1: 8f26f0edddc65cf264dac1530e9f3e943257eee3 # timestamp: 2014-01-12 17:57:11 +0000 # source_branch: http://bzr.squid-cache.org/bzr/squid3/trunk/ # base_revision_id: kinkie@squid-cache.org-20140110190104-\ # 97hv3sj5xm3qvu9g # # Begin patch === modified file 'src/ipc/Makefile.am' --- src/ipc/Makefile.am 2013-01-17 04:25:35 +0000 +++ src/ipc/Makefile.am 2014-01-12 17:15:45 +0000 @@ -13,6 +13,8 @@ Kids.cc \ Kids.h \ Messages.h \ + MemMap.cc \ + MemMap.h \ Queue.cc \ Queue.h \ ReadWriteLock.cc \ === added file 'src/ipc/MemMap.cc' --- src/ipc/MemMap.cc 1970-01-01 00:00:00 +0000 +++ src/ipc/MemMap.cc 2014-01-12 17:15:45 +0000 @@ -0,0 +1,329 @@ +/* + * DEBUG: section 54 Interprocess Communication + */ + +#include "squid.h" +#include "ipc/MemMap.h" +#include "store_key_md5.h" +#include "tools.h" + +Ipc::MemMap::MemMap(const char *const aPath): cleaner(NULL), path(aPath), + shared(shm_old(Shared)(aPath)) +{ + assert(shared->limit > 0); // we should not be created otherwise + debugs(54, 5, "attached map [" << path << "] created: " << + shared->limit); +} + +Ipc::MemMap::Owner * +Ipc::MemMap::Init(const char *const path, const int limit, const size_t extrasSize) +{ + assert(limit > 0); // we should not be created otherwise + Owner *const owner = shm_new(Shared)(path, limit, extrasSize); + debugs(54, 5, "new map [" << path << "] created: " << limit); + return owner; +} + +Ipc::MemMap::Owner * +Ipc::MemMap::Init(const char *const path, const int limit) +{ + return Init(path, limit, 0); +} + +Ipc::MemMap::Slot * +Ipc::MemMap::openForWriting(const cache_key *const key, sfileno &fileno) +{ + debugs(54, 5, "trying to open slot for key " << storeKeyText(key) + << " for writing in map [" << path << ']'); + const int idx = slotIndexByKey(key); + + if (Slot *slot = openForWritingAt(idx)) { + fileno = idx; + return slot; + } + + return NULL; +} + +Ipc::MemMap::Slot * +Ipc::MemMap::openForWritingAt(const sfileno fileno, bool overwriteExisting) +{ + Slot &s = shared->slots[fileno]; + ReadWriteLock &lock = s.lock; + + if (lock.lockExclusive()) { + assert(s.writing() && !s.reading()); + + // bail if we cannot empty this position + if (!s.waitingToBeFreed && !s.empty() && !overwriteExisting) { + lock.unlockExclusive(); + debugs(54, 5, "cannot open existing entry " << fileno << + " for writing " << path); + return NULL; + } + + // free if the entry was used, keeping the entry locked + if (s.waitingToBeFreed || !s.empty()) + freeLocked(s, true); + + assert(s.empty()); + ++shared->count; + + debugs(54, 5, "opened slot at " << fileno << + " for writing in map [" << path << ']'); + return &s; // and keep the entry locked + } + + debugs(54, 5, "failed to open slot at " << fileno << + " for writing in map [" << path << ']'); + return NULL; +} + +void +Ipc::MemMap::closeForWriting(const sfileno fileno, bool lockForReading) +{ + debugs(54, 5, "closing slot at " << fileno << " for writing and " + "openning for reading in map [" << path << ']'); + assert(valid(fileno)); + Slot &s = shared->slots[fileno]; + assert(s.writing()); + if (lockForReading) + s.lock.switchExclusiveToShared(); + else + s.lock.unlockExclusive(); +} + +/// terminate writing the entry, freeing its slot for others to use +void +Ipc::MemMap::abortWriting(const sfileno fileno) +{ + debugs(54, 5, "abort writing slot at " << fileno << + " in map [" << path << ']'); + assert(valid(fileno)); + Slot &s = shared->slots[fileno]; + assert(s.writing()); + freeLocked(s, false); +} + +const Ipc::MemMap::Slot * +Ipc::MemMap::peekAtReader(const sfileno fileno) const +{ + assert(valid(fileno)); + const Slot &s = shared->slots[fileno]; + if (s.reading()) + return &s; // immediate access by lock holder so no locking + if (s.writing()) + return NULL; // cannot read the slot when it is being written + assert(false); // must be locked for reading or writing + return NULL; +} + +void +Ipc::MemMap::free(const sfileno fileno) +{ + debugs(54, 5, "marking slot at " << fileno << " to be freed in" + " map [" << path << ']'); + + assert(valid(fileno)); + Slot &s = shared->slots[fileno]; + + if (s.lock.lockExclusive()) + freeLocked(s, false); + else + s.waitingToBeFreed = true; // mark to free it later +} + +const Ipc::MemMap::Slot * +Ipc::MemMap::openForReading(const cache_key *const key, sfileno &fileno) +{ + debugs(54, 5, "trying to open slot for key " << storeKeyText(key) + << " for reading in map [" << path << ']'); + const int idx = slotIndexByKey(key); + if (const Slot *slot = openForReadingAt(idx)) { + if (slot->sameKey(key)) { + fileno = idx; + debugs(54, 5, "opened slot at " << fileno << " for key " + << storeKeyText(key) << " for reading in map [" << path << + ']'); + return slot; // locked for reading + } + slot->lock.unlockShared(); + } + debugs(54, 5, "failed to open slot for key " << storeKeyText(key) + << " for reading in map [" << path << ']'); + return NULL; +} + +const Ipc::MemMap::Slot * +Ipc::MemMap::openForReadingAt(const sfileno fileno) +{ + debugs(54, 5, "trying to open slot at " << fileno << " for " + "reading in map [" << path << ']'); + assert(valid(fileno)); + Slot &s = shared->slots[fileno]; + + if (!s.lock.lockShared()) { + debugs(54, 5, "failed to lock slot at " << fileno << " for " + "reading in map [" << path << ']'); + return NULL; + } + + if (s.empty()) { + s.lock.unlockShared(); + debugs(54, 7, "empty slot at " << fileno << " for " + "reading in map [" << path << ']'); + return NULL; + } + + if (s.waitingToBeFreed) { + s.lock.unlockShared(); + debugs(54, 7, "dirty slot at " << fileno << " for " + "reading in map [" << path << ']'); + return NULL; + } + + debugs(54, 5, "opened slot at " << fileno << " for reading in" + " map [" << path << ']'); + return &s; +} + +void +Ipc::MemMap::closeForReading(const sfileno fileno) +{ + debugs(54, 5, "closing slot at " << fileno << " for reading in " + "map [" << path << ']'); + assert(valid(fileno)); + Slot &s = shared->slots[fileno]; + assert(s.reading()); + s.lock.unlockShared(); +} + +int +Ipc::MemMap::entryLimit() const +{ + return shared->limit; +} + +int +Ipc::MemMap::entryCount() const +{ + return shared->count; +} + +bool +Ipc::MemMap::full() const +{ + return entryCount() >= entryLimit(); +} + +void +Ipc::MemMap::updateStats(ReadWriteLockStats &stats) const +{ + for (int i = 0; i < shared->limit; ++i) + shared->slots[i].lock.updateStats(stats); +} + +bool +Ipc::MemMap::valid(const int pos) const +{ + return 0 <= pos && pos < entryLimit(); +} + +static +unsigned int +hash_key(const unsigned char *data, unsigned int len, unsigned int hashSize) +{ + unsigned int n; + unsigned int j; + for(j = 0, n = 0; j < len; j++ ) { + n ^= 271 * *data; + ++data; + } + return (n ^ (j * 271)) % hashSize; +} + +int +Ipc::MemMap::slotIndexByKey(const cache_key *const key) const +{ + const unsigned char *k = reinterpret_cast(key); + return hash_key(k, MEMMAP_SLOT_KEY_SIZE, shared->limit); +} + +Ipc::MemMap::Slot & +Ipc::MemMap::slotByKey(const cache_key *const key) +{ + return shared->slots[slotIndexByKey(key)]; +} + +/// unconditionally frees the already exclusively locked slot and releases lock +void +Ipc::MemMap::freeLocked(Slot &s, bool keepLocked) +{ + if (!s.empty() && cleaner) + cleaner->noteFreeMapSlot(&s - shared->slots.raw()); + + s.waitingToBeFreed = false; + memset(s.key, 0, sizeof(s.key)); + if (!keepLocked) + s.lock.unlockExclusive(); + --shared->count; + debugs(54, 5, "freed slot at " << (&s - shared->slots.raw()) << + " in map [" << path << ']'); +} + +/* Ipc::MemMapSlot */ +Ipc::MemMapSlot::MemMapSlot() +{ + memset(key, 0, sizeof(key)); + memset(p, 0, sizeof(p)); + pSize = 0; +} + +void +Ipc::MemMapSlot::set(const unsigned char *aKey, const void *block, size_t blockSize, time_t expireAt) +{ + memcpy(key, aKey, sizeof(key)); + if (block) + memcpy(p, block, blockSize); + pSize = blockSize; + expire = expireAt; +} + +bool +Ipc::MemMapSlot::sameKey(const cache_key *const aKey) const +{ + return (memcmp(key, aKey, sizeof(key)) == 0); +} + +bool +Ipc::MemMapSlot::empty() const +{ + for (unsigned char const*u = key; u < key + sizeof(key); ++u) { + if (*u) + return false; + } + return true; +} + +/* Ipc::MemMap::Shared */ + +Ipc::MemMap::Shared::Shared(const int aLimit, const size_t anExtrasSize): + limit(aLimit), extrasSize(anExtrasSize), count(0), slots(aLimit) +{ +} + +Ipc::MemMap::Shared::~Shared() +{ +} + +size_t +Ipc::MemMap::Shared::sharedMemorySize() const +{ + return SharedMemorySize(limit, extrasSize); +} + +size_t +Ipc::MemMap::Shared::SharedMemorySize(const int limit, const size_t extrasSize) +{ + return sizeof(Shared) + limit * (sizeof(Slot) + extrasSize); +} === added file 'src/ipc/MemMap.h' --- src/ipc/MemMap.h 1970-01-01 00:00:00 +0000 +++ src/ipc/MemMap.h 2014-01-12 17:15:45 +0000 @@ -0,0 +1,140 @@ +#ifndef SQUID_IPC_STORE_MAP_H +#define SQUID_IPC_STORE_MAP_H + +#include "Debug.h" +#include "ipc/mem/FlexibleArray.h" +#include "ipc/mem/Pointer.h" +#include "ipc/ReadWriteLock.h" +#include "SBuf.h" +#include "tools.h" +#include "typedefs.h" + +namespace Ipc +{ + +// The MEMMAP_SLOT_KEY_SIZE and MEMMAP_SLOT_DATA_SIZE must be enough big +// to hold cached keys and data. Currently MemMap used only to store SSL +// shared session data which have keys of 32bytes and at most 10K data +#define MEMMAP_SLOT_KEY_SIZE 32 +#define MEMMAP_SLOT_DATA_SIZE 10*1024 + +/// a MemMap basic element, holding basic shareable memory block info +class MemMapSlot +{ +public: + MemMapSlot(); + size_t size() const {return sizeof(MemMapSlot);} + size_t keySize() const {return sizeof(key);} + bool sameKey(const cache_key *const aKey) const; + void set(const unsigned char *aKey, const void *block, size_t blockSize, time_t expire = 0); + bool empty() const; + bool reading() const { return lock.readers; } + bool writing() const { return lock.writing; } + + Atomic::WordT waitingToBeFreed; ///< may be accessed w/o a lock + mutable ReadWriteLock lock; ///< protects slot data below + unsigned char key[MEMMAP_SLOT_KEY_SIZE]; ///< The entry key + unsigned char p[MEMMAP_SLOT_DATA_SIZE]; ///< The memory block; + size_t pSize; + time_t expire; +}; + +class MemMapCleaner; + +/// A map of MemMapSlots indexed by their keys, with read/write slot locking. +class MemMap +{ +public: + typedef MemMapSlot Slot; + + /// data shared across maps in different processes + class Shared + { + public: + Shared(const int aLimit, const size_t anExtrasSize); + size_t sharedMemorySize() const; + static size_t SharedMemorySize(const int limit, const size_t anExtrasSize); + ~Shared(); + + const int limit; ///< maximum number of map slots + const size_t extrasSize; ///< size of slot extra data + Atomic::Word count; ///< current number of map slots + Ipc::Mem::FlexibleArray slots; ///< storage + }; + +public: + typedef Mem::Owner Owner; + + /// initialize shared memory + static Owner *Init(const char *const path, const int limit); + + MemMap(const char *const aPath); + + /// finds, locks and return a slot for an empty key position, + /// erasing the old entry (if any) + Slot *openForWriting(const cache_key *const key, sfileno &fileno); + + /// locks and returns a slot for the empty fileno position; if + /// overwriteExisting is false and the position is not empty, returns nil + Slot *openForWritingAt(sfileno fileno, bool overwriteExisting = true); + + /// successfully finish writing the entry + void closeForWriting(const sfileno fileno, bool lockForReading = false); + + /// only works on locked entries; returns nil unless the slot is readable + const Slot *peekAtReader(const sfileno fileno) const; + + /// mark the slot as waiting to be freed and, if possible, free it + void free(const sfileno fileno); + + /// open slot for reading, increments read level + const Slot *openForReading(const cache_key *const key, sfileno &fileno); + + /// open slot for reading, increments read level + const Slot *openForReadingAt(const sfileno fileno); + + /// close slot after reading, decrements read level + void closeForReading(const sfileno fileno); + + bool full() const; ///< there are no empty slots left + bool valid(const int n) const; ///< whether n is a valid slot coordinate + int entryCount() const; ///< number of used slots + int entryLimit() const; ///< maximum number of slots that can be used + + /// adds approximate current stats to the supplied ones + void updateStats(ReadWriteLockStats &stats) const; + + /// The cleaner MemMapCleaner::noteFreeMapSlot method called when a + /// readable entry is freed. + MemMapCleaner *cleaner; + +protected: + static Owner *Init(const char *const path, const int limit, const size_t extrasSize); + + const SBuf path; ///< cache_dir path, used for logging + Mem::Pointer shared; + int ttl; + +private: + int slotIndexByKey(const cache_key *const key) const; + Slot &slotByKey(const cache_key *const key); + + Slot *openForReading(Slot &s); + void abortWriting(const sfileno fileno); + void freeIfNeeded(Slot &s); + void freeLocked(Slot &s, bool keepLocked); +}; + +/// API for adjusting external state when dirty map slot is being freed +class MemMapCleaner +{ +public: + virtual ~MemMapCleaner() {} + + /// adjust slot-linked state before a locked Readable slot is erased + virtual void noteFreeMapSlot(const sfileno slotId) = 0; +}; + +} // namespace Ipc + +#endif /* SQUID_IPC_STORE_MAP_H */