Commit ca7d483c104fc4ad3c60f22cb9c95f6b31ed81fe

Authored by Pierre Lassalle
1 parent b5690877
Exists in master

kyotocabinet libs + headers

kyotocabinet-1.2.76/cmdcommon.h 0 → 100644
... ... @@ -0,0 +1,322 @@
  1 +/*************************************************************************************************
  2 + * Common symbols for command line utilities
  3 + * Copyright (C) 2009-2012 FAL Labs
  4 + * This file is part of Kyoto Cabinet.
  5 + * This program is free software: you can redistribute it and/or modify it under the terms of
  6 + * the GNU General Public License as published by the Free Software Foundation, either version
  7 + * 3 of the License, or any later version.
  8 + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
  9 + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  10 + * See the GNU General Public License for more details.
  11 + * You should have received a copy of the GNU General Public License along with this program.
  12 + * If not, see <http://www.gnu.org/licenses/>.
  13 + *************************************************************************************************/
  14 +
  15 +
  16 +#ifndef _CMDCOMMON_H // duplication check
  17 +#define _CMDCOMMON_H
  18 +
  19 +#include <kccommon.h>
  20 +#include <kcutil.h>
  21 +#include <kcthread.h>
  22 +#include <kcfile.h>
  23 +#include <kccompress.h>
  24 +#include <kccompare.h>
  25 +#include <kcmap.h>
  26 +#include <kcregex.h>
  27 +#include <kcdb.h>
  28 +
  29 +#if !defined(_KC_PREFIX)
  30 +#define _KC_PREFIX "*"
  31 +#endif
  32 +#if !defined(_KC_INCLUDEDIR)
  33 +#define _KC_INCLUDEDIR "*"
  34 +#endif
  35 +#if !defined(_KC_LIBDIR)
  36 +#define _KC_LIBDIR "*"
  37 +#endif
  38 +#if !defined(_KC_BINDIR)
  39 +#define _KC_BINDIR "*"
  40 +#endif
  41 +#if !defined(_KC_LIBEXECDIR)
  42 +#define _KC_LIBEXECDIR "*"
  43 +#endif
  44 +#if !defined(_KC_APPINC)
  45 +#define _KC_APPINC "*"
  46 +#endif
  47 +#if !defined(_KC_APPLIBS)
  48 +#define _KC_APPLIBS "*"
  49 +#endif
  50 +
  51 +namespace kc = kyotocabinet;
  52 +
  53 +
  54 +// constants
  55 +const int32_t THREADMAX = 64; // maximum number of threads
  56 +const size_t RECBUFSIZ = 64; // buffer size for a record
  57 +const size_t RECBUFSIZL = 1024; // buffer size for a long record
  58 +
  59 +
  60 +// global variables
  61 +uint64_t g_rnd_x = 123456789;
  62 +uint64_t g_rnd_y = 362436069;
  63 +uint64_t g_rnd_z = 521288629;
  64 +uint64_t g_rnd_w = 88675123;
  65 +
  66 +
  67 +// function prototypes
  68 +void mysrand(int64_t seed);
  69 +int64_t myrand(int64_t range);
  70 +int64_t memusage();
  71 +void oprintf(const char* format, ...);
  72 +void oputchar(char c);
  73 +void eprintf(const char* format, ...);
  74 +void printversion();
  75 +void printdata(const char* buf, int32_t size, bool px);
  76 +bool mygetline(std::istream* is, std::string* str);
  77 +std::string unitnumstr(int64_t num);
  78 +std::string unitnumstrbyte(int64_t num);
  79 +kc::BasicDB::ProgressChecker* stdchecker(const char* prefix, std::ostream* strm);
  80 +kc::BasicDB::Logger* stdlogger(const char* progname, std::ostream* strm);
  81 +void printdb(kc::BasicDB* db, bool px = false);
  82 +
  83 +
  84 +// checker to show progress by printing dots
  85 +class DotChecker : public kc::BasicDB::ProgressChecker {
  86 + public:
  87 + explicit DotChecker(std::ostream* strm, int64_t freq) : strm_(strm), freq_(freq), cnt_(0) {}
  88 + int64_t count() {
  89 + return cnt_;
  90 + }
  91 + private:
  92 + bool check(const char* name, const char* message, int64_t curcnt, int64_t allcnt) {
  93 + if (std::strcmp(message, "processing") || freq_ == 0) return true;
  94 + if (freq_ < 0) {
  95 + cnt_++;
  96 + if (cnt_ % -freq_ == 0) {
  97 + oputchar('.');
  98 + if (cnt_ % (-freq_ * 50) == 0) oprintf(" (%lld)\n", (long long)cnt_);
  99 + }
  100 + } else {
  101 + if (curcnt > cnt_) {
  102 + cnt_ = curcnt;
  103 + if (cnt_ % freq_ == 0) {
  104 + oputchar('.');
  105 + if (cnt_ % (freq_ * 50) == 0) oprintf(" (%lld)\n", (long long)cnt_);
  106 + }
  107 + }
  108 + }
  109 + return true;
  110 + }
  111 + std::ostream* strm_;
  112 + int64_t freq_;
  113 + int64_t cnt_;
  114 +};
  115 +
  116 +
  117 +// get the random seed
  118 +inline void mysrand(int64_t seed) {
  119 + g_rnd_x = seed;
  120 + for (int32_t i = 0; i < 16; i++) {
  121 + myrand(1);
  122 + }
  123 +}
  124 +
  125 +
  126 +// get a random number
  127 +inline int64_t myrand(int64_t range) {
  128 + uint64_t t = g_rnd_x ^ (g_rnd_x << 11);
  129 + g_rnd_x = g_rnd_y;
  130 + g_rnd_y = g_rnd_z;
  131 + g_rnd_z = g_rnd_w;
  132 + g_rnd_w = (g_rnd_w ^ (g_rnd_w >> 19)) ^ (t ^ (t >> 8));
  133 + return (g_rnd_w & kc::INT64MAX) % range;
  134 +}
  135 +
  136 +
  137 +// get the current memory usage
  138 +inline int64_t memusage() {
  139 + std::map<std::string, std::string> info;
  140 + kc::getsysinfo(&info);
  141 + return kc::atoi(info["mem_rss"].c_str());
  142 +}
  143 +
  144 +
  145 +// print formatted information string and flush the buffer
  146 +inline void oprintf(const char* format, ...) {
  147 + std::string msg;
  148 + va_list ap;
  149 + va_start(ap, format);
  150 + kc::vstrprintf(&msg, format, ap);
  151 + va_end(ap);
  152 + std::cout << msg;
  153 + std::cout.flush();
  154 +}
  155 +
  156 +
  157 +// print a character and flush the buffer
  158 +inline void oputchar(char c) {
  159 + std::cout << c;
  160 + std::cout.flush();
  161 +}
  162 +
  163 +
  164 +// print formatted error string and flush the buffer
  165 +inline void eprintf(const char* format, ...) {
  166 + std::string msg;
  167 + va_list ap;
  168 + va_start(ap, format);
  169 + kc::vstrprintf(&msg, format, ap);
  170 + va_end(ap);
  171 + std::cerr << msg;
  172 + std::cerr.flush();
  173 +}
  174 +
  175 +
  176 +// print the versin information
  177 +inline void printversion() {
  178 + oprintf("Kyoto Cabinet %s (%d.%d:%d) on %s\n",
  179 + kc::VERSION, kc::LIBVER, kc::LIBREV, kc::FMTVER, kc::OSNAME);
  180 +}
  181 +
  182 +
  183 +// print record data
  184 +inline void printdata(const char* buf, int32_t size, bool px) {
  185 + size_t cnt = 0;
  186 + char numbuf[kc::NUMBUFSIZ];
  187 + while (size-- > 0) {
  188 + if (px) {
  189 + if (cnt++ > 0) putchar(' ');
  190 + std::sprintf(numbuf, "%02X", *(unsigned char*)buf);
  191 + std::cout << numbuf;
  192 + } else {
  193 + std::cout << *buf;
  194 + }
  195 + buf++;
  196 + }
  197 +}
  198 +
  199 +
  200 +// read a line from a file descriptor
  201 +inline bool mygetline(std::istream* is, std::string* str) {
  202 + str->clear();
  203 + bool hit = false;
  204 + char c;
  205 + while (is->get(c)) {
  206 + hit = true;
  207 + if (c == '\0' || c == '\r') continue;
  208 + if (c == '\n') break;
  209 + str->append(1, c);
  210 + }
  211 + return hit;
  212 +}
  213 +
  214 +
  215 +// convert a number into the string with the decimal unit
  216 +inline std::string unitnumstr(int64_t num) {
  217 + if (num >= std::pow(1000.0, 6)) {
  218 + return kc::strprintf("%.3Lf quintillion", (long double)num / std::pow(1000.0, 6));
  219 + } else if (num >= std::pow(1000.0, 5)) {
  220 + return kc::strprintf("%.3Lf quadrillion", (long double)num / std::pow(1000.0, 5));
  221 + } else if (num >= std::pow(1000.0, 4)) {
  222 + return kc::strprintf("%.3Lf trillion", (long double)num / std::pow(1000.0, 4));
  223 + } else if (num >= std::pow(1000.0, 3)) {
  224 + return kc::strprintf("%.3Lf billion", (long double)num / std::pow(1000.0, 3));
  225 + } else if (num >= std::pow(1000.0, 2)) {
  226 + return kc::strprintf("%.3Lf million", (long double)num / std::pow(1000.0, 2));
  227 + } else if (num >= std::pow(1000.0, 1)) {
  228 + return kc::strprintf("%.3Lf thousand", (long double)num / std::pow(1000.0, 1));
  229 + }
  230 + return kc::strprintf("%lld", (long long)num);
  231 +}
  232 +
  233 +
  234 +// convert a number into the string with the byte unit
  235 +inline std::string unitnumstrbyte(int64_t num) {
  236 + if ((unsigned long long)num >= 1ULL << 60) {
  237 + return kc::strprintf("%.3Lf EiB", (long double)num / (1ULL << 60));
  238 + } else if ((unsigned long long)num >= 1ULL << 50) {
  239 + return kc::strprintf("%.3Lf PiB", (long double)num / (1ULL << 50));
  240 + } else if ((unsigned long long)num >= 1ULL << 40) {
  241 + return kc::strprintf("%.3Lf TiB", (long double)num / (1ULL << 40));
  242 + } else if ((unsigned long long)num >= 1ULL << 30) {
  243 + return kc::strprintf("%.3Lf GiB", (long double)num / (1ULL << 30));
  244 + } else if ((unsigned long long)num >= 1ULL << 20) {
  245 + return kc::strprintf("%.3Lf MiB", (long double)num / (1ULL << 20));
  246 + } else if ((unsigned long long)num >= 1ULL << 10) {
  247 + return kc::strprintf("%.3Lf KiB", (long double)num / (1ULL << 10));
  248 + }
  249 + return kc::strprintf("%lld B", (long long)num);
  250 +}
  251 +
  252 +
  253 +// get the progress checker to print the parameters
  254 +inline kc::BasicDB::ProgressChecker* stdchecker(const char* prefix, std::ostream* strm) {
  255 + class CheckerImpl : public kc::BasicDB::ProgressChecker {
  256 + public:
  257 + explicit CheckerImpl(std::ostream* strm, const char* prefix) :
  258 + strm_(strm), prefix_(prefix) {}
  259 + bool check(const char* name, const char* message, int64_t curcnt, int64_t allcnt) {
  260 + *strm_ << prefix_ << ": " << name << ": " << message << ": " <<
  261 + curcnt << "/" << allcnt << std::endl;
  262 + return true;
  263 + }
  264 + private:
  265 + std::ostream* strm_;
  266 + const char* prefix_;
  267 + };
  268 + static CheckerImpl checker(strm, prefix);
  269 + return &checker;
  270 +}
  271 +
  272 +
  273 +// get the logger into the standard stream
  274 +inline kc::BasicDB::Logger* stdlogger(const char* prefix, std::ostream* strm) {
  275 + class LoggerImpl : public kc::BasicDB::Logger {
  276 + public:
  277 + explicit LoggerImpl(std::ostream* strm, const char* prefix) :
  278 + strm_(strm), prefix_(prefix) {}
  279 + void log(const char* file, int32_t line, const char* func, Kind kind,
  280 + const char* message) {
  281 + const char* kstr = "MISC";
  282 + switch (kind) {
  283 + case kc::BasicDB::Logger::DEBUG: kstr = "DEBUG"; break;
  284 + case kc::BasicDB::Logger::INFO: kstr = "INFO"; break;
  285 + case kc::BasicDB::Logger::WARN: kstr = "WARN"; break;
  286 + case kc::BasicDB::Logger::ERROR: kstr = "ERROR"; break;
  287 + }
  288 + *strm_ << prefix_ << ": [" << kstr << "]: " <<
  289 + file << ": " << line << ": " << func << ": " << message << std::endl;
  290 + }
  291 + private:
  292 + std::ostream* strm_;
  293 + const char* prefix_;
  294 + };
  295 + static LoggerImpl logger(strm, prefix);
  296 + return &logger;
  297 +}
  298 +
  299 +
  300 +// print all record of a database
  301 +inline void printdb(kc::BasicDB* db, bool px) {
  302 + class Printer : public kc::DB::Visitor {
  303 + public:
  304 + explicit Printer(bool px) : px_(px) {}
  305 + private:
  306 + const char* visit_full(const char* kbuf, size_t ksiz,
  307 + const char* vbuf, size_t vsiz, size_t* sp) {
  308 + printdata(kbuf, ksiz, px_);
  309 + oputchar('\t');
  310 + printdata(vbuf, vsiz, px_);
  311 + oputchar('\n');
  312 + return NOP;
  313 + }
  314 + bool px_;
  315 + } printer(px);
  316 + db->iterate(&printer, false);
  317 +}
  318 +
  319 +
  320 +#endif // duplication check
  321 +
  322 +// END OF FILE
... ...
kyotocabinet-1.2.76/kccachedb.h 0 → 100644
... ... @@ -0,0 +1,2067 @@
  1 +/*************************************************************************************************
  2 + * Cache hash database
  3 + * Copyright (C) 2009-2012 FAL Labs
  4 + * This file is part of Kyoto Cabinet.
  5 + * This program is free software: you can redistribute it and/or modify it under the terms of
  6 + * the GNU General Public License as published by the Free Software Foundation, either version
  7 + * 3 of the License, or any later version.
  8 + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
  9 + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  10 + * See the GNU General Public License for more details.
  11 + * You should have received a copy of the GNU General Public License along with this program.
  12 + * If not, see <http://www.gnu.org/licenses/>.
  13 + *************************************************************************************************/
  14 +
  15 +
  16 +#ifndef _KCCACHEDB_H // duplication check
  17 +#define _KCCACHEDB_H
  18 +
  19 +#include <kccommon.h>
  20 +#include <kcutil.h>
  21 +#include <kcthread.h>
  22 +#include <kcfile.h>
  23 +#include <kccompress.h>
  24 +#include <kccompare.h>
  25 +#include <kcmap.h>
  26 +#include <kcregex.h>
  27 +#include <kcdb.h>
  28 +#include <kcplantdb.h>
  29 +
  30 +namespace kyotocabinet { // common namespace
  31 +
  32 +
  33 +/**
  34 + * On-memory hash database with LRU deletion.
  35 + * @note This class is a concrete class to operate a hash database on memory. This class can be
  36 + * inherited but overwriting methods is forbidden. Before every database operation, it is
  37 + * necessary to call the CacheDB::open method in order to open a database file and connect the
  38 + * database object to it. To avoid data missing or corruption, it is important to close every
  39 + * database file by the CacheDB::close method when the database is no longer in use. It is
  40 + * forbidden for multible database objects in a process to open the same database at the same
  41 + * time. It is forbidden to share a database object with child processes.
  42 + */
  43 +class CacheDB : public BasicDB {
  44 + friend class PlantDB<CacheDB, BasicDB::TYPEGRASS>;
  45 + public:
  46 + class Cursor;
  47 + private:
  48 + struct Record;
  49 + struct TranLog;
  50 + struct Slot;
  51 + class Repeater;
  52 + class Setter;
  53 + class Remover;
  54 + class ScopedVisitor;
  55 + /** An alias of list of cursors. */
  56 + typedef std::list<Cursor*> CursorList;
  57 + /** An alias of list of transaction logs. */
  58 + typedef std::list<TranLog> TranLogList;
  59 + /** The number of slot tables. */
  60 + static const int32_t SLOTNUM = 16;
  61 + /** The default bucket number. */
  62 + static const size_t DEFBNUM = 1048583LL;
  63 + /** The mininum number of buckets to use mmap. */
  64 + static const size_t ZMAPBNUM = 32768;
  65 + /** The maximum size of each key. */
  66 + static const uint32_t KSIZMAX = 0xfffff;
  67 + /** The size of the record buffer. */
  68 + static const size_t RECBUFSIZ = 48;
  69 + /** The size of the opaque buffer. */
  70 + static const size_t OPAQUESIZ = 16;
  71 + /** The threshold of busy loop and sleep for locking. */
  72 + static const uint32_t LOCKBUSYLOOP = 8192;
  73 + public:
  74 + /**
  75 + * Cursor to indicate a record.
  76 + */
  77 + class Cursor : public BasicDB::Cursor {
  78 + friend class CacheDB;
  79 + public:
  80 + /**
  81 + * Constructor.
  82 + * @param db the container database object.
  83 + */
  84 + explicit Cursor(CacheDB* db) : db_(db), sidx_(-1), rec_(NULL) {
  85 + _assert_(db);
  86 + ScopedRWLock lock(&db_->mlock_, true);
  87 + db_->curs_.push_back(this);
  88 + }
  89 + /**
  90 + * Destructor.
  91 + */
  92 + virtual ~Cursor() {
  93 + _assert_(true);
  94 + if (!db_) return;
  95 + ScopedRWLock lock(&db_->mlock_, true);
  96 + db_->curs_.remove(this);
  97 + }
  98 + /**
  99 + * Accept a visitor to the current record.
  100 + * @param visitor a visitor object.
  101 + * @param writable true for writable operation, or false for read-only operation.
  102 + * @param step true to move the cursor to the next record, or false for no move.
  103 + * @return true on success, or false on failure.
  104 + * @note The operation for each record is performed atomically and other threads accessing
  105 + * the same record are blocked. To avoid deadlock, any explicit database operation must not
  106 + * be performed in this function.
  107 + */
  108 + bool accept(Visitor* visitor, bool writable = true, bool step = false) {
  109 + _assert_(visitor);
  110 + ScopedRWLock lock(&db_->mlock_, true);
  111 + if (db_->omode_ == 0) {
  112 + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
  113 + return false;
  114 + }
  115 + if (writable && !(db_->omode_ & OWRITER)) {
  116 + db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
  117 + return false;
  118 + }
  119 + if (sidx_ < 0 || !rec_) {
  120 + db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
  121 + return false;
  122 + }
  123 + uint32_t rksiz = rec_->ksiz & KSIZMAX;
  124 + char* dbuf = (char*)rec_ + sizeof(*rec_);
  125 + const char* rvbuf = dbuf + rksiz;
  126 + size_t rvsiz = rec_->vsiz;
  127 + char* zbuf = NULL;
  128 + size_t zsiz = 0;
  129 + if (db_->comp_) {
  130 + zbuf = db_->comp_->decompress(rvbuf, rvsiz, &zsiz);
  131 + if (zbuf) {
  132 + rvbuf = zbuf;
  133 + rvsiz = zsiz;
  134 + }
  135 + }
  136 + size_t vsiz;
  137 + const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vsiz);
  138 + delete[] zbuf;
  139 + if (vbuf == Visitor::REMOVE) {
  140 + uint64_t hash = db_->hash_record(dbuf, rksiz) / SLOTNUM;
  141 + Slot* slot = db_->slots_ + sidx_;
  142 + Repeater repeater(Visitor::REMOVE, 0);
  143 + db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, false);
  144 + } else if (vbuf == Visitor::NOP) {
  145 + if (step) step_impl();
  146 + } else {
  147 + uint64_t hash = db_->hash_record(dbuf, rksiz) / SLOTNUM;
  148 + Slot* slot = db_->slots_ + sidx_;
  149 + Repeater repeater(vbuf, vsiz);
  150 + db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, false);
  151 + if (step) step_impl();
  152 + }
  153 + return true;
  154 + }
  155 + /**
  156 + * Jump the cursor to the first record for forward scan.
  157 + * @return true on success, or false on failure.
  158 + */
  159 + bool jump() {
  160 + _assert_(true);
  161 + ScopedRWLock lock(&db_->mlock_, true);
  162 + if (db_->omode_ == 0) {
  163 + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
  164 + return false;
  165 + }
  166 + for (int32_t i = 0; i < SLOTNUM; i++) {
  167 + Slot* slot = db_->slots_ + i;
  168 + if (slot->first) {
  169 + sidx_ = i;
  170 + rec_ = slot->first;
  171 + return true;
  172 + }
  173 + }
  174 + db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
  175 + sidx_ = -1;
  176 + rec_ = NULL;
  177 + return false;
  178 + }
  179 + /**
  180 + * Jump the cursor to a record for forward scan.
  181 + * @param kbuf the pointer to the key region.
  182 + * @param ksiz the size of the key region.
  183 + * @return true on success, or false on failure.
  184 + */
  185 + bool jump(const char* kbuf, size_t ksiz) {
  186 + _assert_(kbuf && ksiz <= MEMMAXSIZ);
  187 + ScopedRWLock lock(&db_->mlock_, true);
  188 + if (db_->omode_ == 0) {
  189 + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
  190 + return false;
  191 + }
  192 + if (ksiz > KSIZMAX) ksiz = KSIZMAX;
  193 + uint64_t hash = db_->hash_record(kbuf, ksiz);
  194 + int32_t sidx = hash % SLOTNUM;
  195 + hash /= SLOTNUM;
  196 + Slot* slot = db_->slots_ + sidx;
  197 + size_t bidx = hash % slot->bnum;
  198 + Record* rec = slot->buckets[bidx];
  199 + Record** entp = slot->buckets + bidx;
  200 + uint32_t fhash = db_->fold_hash(hash) & ~KSIZMAX;
  201 + while (rec) {
  202 + uint32_t rhash = rec->ksiz & ~KSIZMAX;
  203 + uint32_t rksiz = rec->ksiz & KSIZMAX;
  204 + if (fhash > rhash) {
  205 + entp = &rec->left;
  206 + rec = rec->left;
  207 + } else if (fhash < rhash) {
  208 + entp = &rec->right;
  209 + rec = rec->right;
  210 + } else {
  211 + char* dbuf = (char*)rec + sizeof(*rec);
  212 + int32_t kcmp = db_->compare_keys(kbuf, ksiz, dbuf, rksiz);
  213 + if (kcmp < 0) {
  214 + entp = &rec->left;
  215 + rec = rec->left;
  216 + } else if (kcmp > 0) {
  217 + entp = &rec->right;
  218 + rec = rec->right;
  219 + } else {
  220 + sidx_ = sidx;
  221 + rec_ = rec;
  222 + return true;
  223 + }
  224 + }
  225 + }
  226 + db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
  227 + sidx_ = -1;
  228 + rec_ = NULL;
  229 + return false;
  230 + }
  231 + /**
  232 + * Jump the cursor to a record for forward scan.
  233 + * @note Equal to the original Cursor::jump method except that the parameter is std::string.
  234 + */
  235 + bool jump(const std::string& key) {
  236 + _assert_(true);
  237 + return jump(key.c_str(), key.size());
  238 + }
  239 + /**
  240 + * Jump the cursor to the last record for backward scan.
  241 + * @note This is a dummy implementation for compatibility.
  242 + */
  243 + bool jump_back() {
  244 + _assert_(true);
  245 + ScopedRWLock lock(&db_->mlock_, true);
  246 + if (db_->omode_ == 0) {
  247 + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
  248 + return false;
  249 + }
  250 + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
  251 + return false;
  252 + }
  253 + /**
  254 + * Jump the cursor to a record for backward scan.
  255 + * @note This is a dummy implementation for compatibility.
  256 + */
  257 + bool jump_back(const char* kbuf, size_t ksiz) {
  258 + _assert_(kbuf && ksiz <= MEMMAXSIZ);
  259 + ScopedRWLock lock(&db_->mlock_, true);
  260 + if (db_->omode_ == 0) {
  261 + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
  262 + return false;
  263 + }
  264 + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
  265 + return false;
  266 + }
  267 + /**
  268 + * Jump the cursor to a record for backward scan.
  269 + * @note This is a dummy implementation for compatibility.
  270 + */
  271 + bool jump_back(const std::string& key) {
  272 + _assert_(true);
  273 + ScopedRWLock lock(&db_->mlock_, true);
  274 + if (db_->omode_ == 0) {
  275 + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
  276 + return false;
  277 + }
  278 + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
  279 + return false;
  280 + }
  281 + /**
  282 + * Step the cursor to the next record.
  283 + * @return true on success, or false on failure.
  284 + */
  285 + bool step() {
  286 + _assert_(true);
  287 + ScopedRWLock lock(&db_->mlock_, true);
  288 + if (db_->omode_ == 0) {
  289 + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
  290 + return false;
  291 + }
  292 + if (sidx_ < 0 || !rec_) {
  293 + db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
  294 + return false;
  295 + }
  296 + bool err = false;
  297 + if (!step_impl()) err = true;
  298 + return !err;
  299 + }
  300 + /**
  301 + * Step the cursor to the previous record.
  302 + * @note This is a dummy implementation for compatibility.
  303 + */
  304 + bool step_back() {
  305 + _assert_(true);
  306 + ScopedRWLock lock(&db_->mlock_, true);
  307 + if (db_->omode_ == 0) {
  308 + db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
  309 + return false;
  310 + }
  311 + db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
  312 + return false;
  313 + }
  314 + /**
  315 + * Get the database object.
  316 + * @return the database object.
  317 + */
  318 + CacheDB* db() {
  319 + _assert_(true);
  320 + return db_;
  321 + }
  322 + private:
  323 + /**
  324 + * Step the cursor to the next record.
  325 + * @return true on success, or false on failure.
  326 + */
  327 + bool step_impl() {
  328 + _assert_(true);
  329 + rec_ = rec_->next;
  330 + if (!rec_) {
  331 + for (int32_t i = sidx_ + 1; i < SLOTNUM; i++) {
  332 + Slot* slot = db_->slots_ + i;
  333 + if (slot->first) {
  334 + sidx_ = i;
  335 + rec_ = slot->first;
  336 + return true;
  337 + }
  338 + }
  339 + db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
  340 + sidx_ = -1;
  341 + rec_ = NULL;
  342 + return false;
  343 + }
  344 + return true;
  345 + }
  346 + /** Dummy constructor to forbid the use. */
  347 + Cursor(const Cursor&);
  348 + /** Dummy Operator to forbid the use. */
  349 + Cursor& operator =(const Cursor&);
  350 + /** The inner database. */
  351 + CacheDB* db_;
  352 + /** The index of the current slot. */
  353 + int32_t sidx_;
  354 + /** The current record. */
  355 + Record* rec_;
  356 + };
  357 + /**
  358 + * Tuning options.
  359 + */
  360 + enum Option {
  361 + TSMALL = 1 << 0, ///< dummy for compatibility
  362 + TLINEAR = 1 << 1, ///< dummy for compatibility
  363 + TCOMPRESS = 1 << 2 ///< compress each record
  364 + };
  365 + /**
  366 + * Status flags.
  367 + */
  368 + enum Flag {
  369 + FOPEN = 1 << 0, ///< dummy for compatibility
  370 + FFATAL = 1 << 1 ///< dummy for compatibility
  371 + };
  372 + /**
  373 + * Default constructor.
  374 + */
  375 + explicit CacheDB() :
  376 + mlock_(), flock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NULL),
  377 + omode_(0), curs_(), path_(""), type_(TYPECACHE),
  378 + opts_(0), bnum_(DEFBNUM), capcnt_(-1), capsiz_(-1),
  379 + opaque_(), embcomp_(ZLIBRAWCOMP), comp_(NULL), slots_(), rttmode_(true), tran_(false) {
  380 + _assert_(true);
  381 + }
  382 + /**
  383 + * Destructor.
  384 + * @note If the database is not closed, it is closed implicitly.
  385 + */
  386 + virtual ~CacheDB() {
  387 + _assert_(true);
  388 + if (omode_ != 0) close();
  389 + if (!curs_.empty()) {
  390 + CursorList::const_iterator cit = curs_.begin();
  391 + CursorList::const_iterator citend = curs_.end();
  392 + while (cit != citend) {
  393 + Cursor* cur = *cit;
  394 + cur->db_ = NULL;
  395 + ++cit;
  396 + }
  397 + }
  398 + }
  399 + /**
  400 + * Accept a visitor to a record.
  401 + * @param kbuf the pointer to the key region.
  402 + * @param ksiz the size of the key region.
  403 + * @param visitor a visitor object.
  404 + * @param writable true for writable operation, or false for read-only operation.
  405 + * @return true on success, or false on failure.
  406 + * @note The operation for each record is performed atomically and other threads accessing the
  407 + * same record are blocked. To avoid deadlock, any explicit database operation must not be
  408 + * performed in this function.
  409 + */
  410 + bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) {
  411 + _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor);
  412 + ScopedRWLock lock(&mlock_, false);
  413 + if (omode_ == 0) {
  414 + set_error(_KCCODELINE_, Error::INVALID, "not opened");
  415 + return false;
  416 + }
  417 + if (writable && !(omode_ & OWRITER)) {
  418 + set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
  419 + return false;
  420 + }
  421 + if (ksiz > KSIZMAX) ksiz = KSIZMAX;
  422 + uint64_t hash = hash_record(kbuf, ksiz);
  423 + int32_t sidx = hash % SLOTNUM;
  424 + hash /= SLOTNUM;
  425 + Slot* slot = slots_ + sidx;
  426 + slot->lock.lock();
  427 + accept_impl(slot, hash, kbuf, ksiz, visitor, comp_, rttmode_);
  428 + slot->lock.unlock();
  429 + return true;
  430 + }
  431 + /**
  432 + * Accept a visitor to multiple records at once.
  433 + * @param keys specifies a string vector of the keys.
  434 + * @param visitor a visitor object.
  435 + * @param writable true for writable operation, or false for read-only operation.
  436 + * @return true on success, or false on failure.
  437 + * @note The operations for specified records are performed atomically and other threads
  438 + * accessing the same records are blocked. To avoid deadlock, any explicit database operation
  439 + * must not be performed in this function.
  440 + */
  441 + bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor,
  442 + bool writable = true) {
  443 + _assert_(visitor);
  444 + ScopedRWLock lock(&mlock_, false);
  445 + if (omode_ == 0) {
  446 + set_error(_KCCODELINE_, Error::INVALID, "not opened");
  447 + return false;
  448 + }
  449 + if (writable && !(omode_ & OWRITER)) {
  450 + set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
  451 + return false;
  452 + }
  453 + ScopedVisitor svis(visitor);
  454 + size_t knum = keys.size();
  455 + if (knum < 1) return true;
  456 + struct RecordKey {
  457 + const char* kbuf;
  458 + size_t ksiz;
  459 + uint64_t hash;
  460 + int32_t sidx;
  461 + };
  462 + RecordKey* rkeys = new RecordKey[knum];
  463 + std::set<int32_t> sidxs;
  464 + for (size_t i = 0; i < knum; i++) {
  465 + const std::string& key = keys[i];
  466 + RecordKey* rkey = rkeys + i;
  467 + rkey->kbuf = key.data();
  468 + rkey->ksiz = key.size();
  469 + if (rkey->ksiz > KSIZMAX) rkey->ksiz = KSIZMAX;
  470 + rkey->hash = hash_record(rkey->kbuf, rkey->ksiz);
  471 + rkey->sidx = rkey->hash % SLOTNUM;
  472 + sidxs.insert(rkey->sidx);
  473 + rkey->hash /= SLOTNUM;
  474 + }
  475 + std::set<int32_t>::iterator sit = sidxs.begin();
  476 + std::set<int32_t>::iterator sitend = sidxs.end();
  477 + while (sit != sitend) {
  478 + Slot* slot = slots_ + *sit;
  479 + slot->lock.lock();
  480 + ++sit;
  481 + }
  482 + for (size_t i = 0; i < knum; i++) {
  483 + RecordKey* rkey = rkeys + i;
  484 + Slot* slot = slots_ + rkey->sidx;
  485 + accept_impl(slot, rkey->hash, rkey->kbuf, rkey->ksiz, visitor, comp_, rttmode_);
  486 + }
  487 + sit = sidxs.begin();
  488 + sitend = sidxs.end();
  489 + while (sit != sitend) {
  490 + Slot* slot = slots_ + *sit;
  491 + slot->lock.unlock();
  492 + ++sit;
  493 + }
  494 + delete[] rkeys;
  495 + return true;
  496 + }
  497 + /**
  498 + * Iterate to accept a visitor for each record.
  499 + * @param visitor a visitor object.
  500 + * @param writable true for writable operation, or false for read-only operation.
  501 + * @param checker a progress checker object. If it is NULL, no checking is performed.
  502 + * @return true on success, or false on failure.
  503 + * @note The whole iteration is performed atomically and other threads are blocked. To avoid
  504 + * deadlock, any explicit database operation must not be performed in this function.
  505 + */
  506 + bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* checker = NULL) {
  507 + _assert_(visitor);
  508 + ScopedRWLock lock(&mlock_, true);
  509 + if (omode_ == 0) {
  510 + set_error(_KCCODELINE_, Error::INVALID, "not opened");
  511 + return false;
  512 + }
  513 + if (writable && !(omode_ & OWRITER)) {
  514 + set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
  515 + return false;
  516 + }
  517 + ScopedVisitor svis(visitor);
  518 + int64_t allcnt = count_impl();
  519 + if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
  520 + set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
  521 + return false;
  522 + }
  523 + int64_t curcnt = 0;
  524 + for (int32_t i = 0; i < SLOTNUM; i++) {
  525 + Slot* slot = slots_ + i;
  526 + Record* rec = slot->first;
  527 + while (rec) {
  528 + Record* next = rec->next;
  529 + uint32_t rksiz = rec->ksiz & KSIZMAX;
  530 + char* dbuf = (char*)rec + sizeof(*rec);
  531 + const char* rvbuf = dbuf + rksiz;
  532 + size_t rvsiz = rec->vsiz;
  533 + char* zbuf = NULL;
  534 + size_t zsiz = 0;
  535 + if (comp_) {
  536 + zbuf = comp_->decompress(rvbuf, rvsiz, &zsiz);
  537 + if (zbuf) {
  538 + rvbuf = zbuf;
  539 + rvsiz = zsiz;
  540 + }
  541 + }
  542 + size_t vsiz;
  543 + const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vsiz);
  544 + delete[] zbuf;
  545 + if (vbuf == Visitor::REMOVE) {
  546 + uint64_t hash = hash_record(dbuf, rksiz) / SLOTNUM;
  547 + Repeater repeater(Visitor::REMOVE, 0);
  548 + accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, false);
  549 + } else if (vbuf != Visitor::NOP) {
  550 + uint64_t hash = hash_record(dbuf, rksiz) / SLOTNUM;
  551 + Repeater repeater(vbuf, vsiz);
  552 + accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, false);
  553 + }
  554 + rec = next;
  555 + curcnt++;
  556 + if (checker && !checker->check("iterate", "processing", curcnt, allcnt)) {
  557 + set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
  558 + return false;
  559 + }
  560 + }
  561 + }
  562 + if (checker && !checker->check("iterate", "ending", -1, allcnt)) {
  563 + set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
  564 + return false;
  565 + }
  566 + trigger_meta(MetaTrigger::ITERATE, "iterate");
  567 + return true;
  568 + }
  569 + /**
  570 + * Scan each record in parallel.
  571 + * @param visitor a visitor object.
  572 + * @param thnum the number of worker threads.
  573 + * @param checker a progress checker object. If it is NULL, no checking is performed.
  574 + * @return true on success, or false on failure.
  575 + * @note This function is for reading records and not for updating ones. The return value of
  576 + * the visitor is just ignored. To avoid deadlock, any explicit database operation must not
  577 + * be performed in this function.
  578 + */
  579 + bool scan_parallel(Visitor *visitor, size_t thnum, ProgressChecker* checker = NULL) {
  580 + _assert_(visitor && thnum <= MEMMAXSIZ);
  581 + ScopedRWLock lock(&mlock_, false);
  582 + if (omode_ == 0) {
  583 + set_error(_KCCODELINE_, Error::INVALID, "not opened");
  584 + return false;
  585 + }
  586 + if (thnum < 1) thnum = 1;
  587 + thnum = std::pow(2.0, (int32_t)(std::log(thnum * std::sqrt(2.0)) / std::log(2.0)));
  588 + if (thnum > (size_t)SLOTNUM) thnum = SLOTNUM;
  589 + ScopedVisitor svis(visitor);
  590 + int64_t allcnt = count_impl();
  591 + if (checker && !checker->check("scan_parallel", "beginning", -1, allcnt)) {
  592 + set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
  593 + return false;
  594 + }
  595 + class ThreadImpl : public Thread {
  596 + public:
  597 + explicit ThreadImpl() :
  598 + db_(NULL), visitor_(NULL), checker_(NULL), allcnt_(0), slots_(), error_() {}
  599 + void init(CacheDB* db, Visitor* visitor, ProgressChecker* checker, int64_t allcnt) {
  600 + db_ = db;
  601 + visitor_ = visitor;
  602 + checker_ = checker;
  603 + allcnt_ = allcnt;
  604 + }
  605 + void add_slot(Slot* slot) {
  606 + slots_.push_back(slot);
  607 + }
  608 + const Error& error() {
  609 + return error_;
  610 + }
  611 + private:
  612 + void run() {
  613 + CacheDB* db = db_;
  614 + Visitor* visitor = visitor_;
  615 + ProgressChecker* checker = checker_;
  616 + int64_t allcnt = allcnt_;
  617 + Compressor* comp = db->comp_;
  618 + std::vector<Slot*>::iterator sit = slots_.begin();
  619 + std::vector<Slot*>::iterator sitend = slots_.end();
  620 + while (sit != sitend) {
  621 + Slot* slot = *sit;
  622 + Record* rec = slot->first;
  623 + while (rec) {
  624 + Record* next = rec->next;
  625 + uint32_t rksiz = rec->ksiz & KSIZMAX;
  626 + char* dbuf = (char*)rec + sizeof(*rec);
  627 + const char* rvbuf = dbuf + rksiz;
  628 + size_t rvsiz = rec->vsiz;
  629 + char* zbuf = NULL;
  630 + size_t zsiz = 0;
  631 + if (comp) {
  632 + zbuf = comp->decompress(rvbuf, rvsiz, &zsiz);
  633 + if (zbuf) {
  634 + rvbuf = zbuf;
  635 + rvsiz = zsiz;
  636 + }
  637 + }
  638 + size_t vsiz;
  639 + visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vsiz);
  640 + delete[] zbuf;
  641 + rec = next;
  642 + if (checker && !checker->check("scan_parallel", "processing", -1, allcnt)) {
  643 + db->set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
  644 + error_ = db->error();
  645 + break;
  646 + }
  647 + }
  648 + ++sit;
  649 + }
  650 + }
  651 + CacheDB* db_;
  652 + Visitor* visitor_;
  653 + ProgressChecker* checker_;
  654 + int64_t allcnt_;
  655 + std::vector<Slot*> slots_;
  656 + Error error_;
  657 + };
  658 + bool err = false;
  659 + bool orttmode = rttmode_;
  660 + rttmode_ = false;
  661 + ThreadImpl* threads = new ThreadImpl[thnum];
  662 + for (int32_t i = 0; i < SLOTNUM; i++) {
  663 + ThreadImpl* thread = threads + (i % thnum);
  664 + thread->add_slot(slots_ + i);
  665 + }
  666 + for (size_t i = 0; i < thnum; i++) {
  667 + ThreadImpl* thread = threads + i;
  668 + thread->init(this, visitor, checker, allcnt);
  669 + thread->start();
  670 + }
  671 + for (size_t i = 0; i < thnum; i++) {
  672 + ThreadImpl* thread = threads + i;
  673 + thread->join();
  674 + if (thread->error() != Error::SUCCESS) {
  675 + *error_ = thread->error();
  676 + err = true;
  677 + }
  678 + }
  679 + delete[] threads;
  680 + rttmode_ = orttmode;
  681 + if (err) return false;
  682 + if (checker && !checker->check("scan_parallel", "ending", -1, allcnt)) {
  683 + set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
  684 + return false;
  685 + }
  686 + trigger_meta(MetaTrigger::ITERATE, "scan_parallel");
  687 + return true;
  688 + }
  689 + /**
  690 + * Get the last happened error.