Hatena::Grouptechmemo

msyktの日記

2011-08-11TokyoCabinetのTDBのコードを読む

[TCTDB]tctdbnew

21:33

/* Create a table database object. */
TCTDB *tctdbnew(void){
  TCTDB *tdb;
  TCMALLOC(tdb, sizeof(*tdb));
  tctdbclear(tdb);
  tdb->hdb = tchdbnew();
  tchdbtune(tdb->hdb, TDBDEFBNUM, TDBDEFAPOW, TDBDEFFPOW, 0);
  tchdbsetxmsiz(tdb->hdb, TDBDEFXMSIZ);
  return tdb;
}
  1. TDB用のヒープに領域を確保する(TCMALLOCはmalloc)
  2. TDBの各値を初期化
  3. TDBで保持するhash databaseを作成
  4. hash databaseのパラメータを設定
  5. hash databaseのextra mapped memoryのsizeを設定(アラインメントされる)

[TCTDB]tctdbdel

21:46

/* Delete a table database object. */
void tctdbdel(TCTDB *tdb){
  assert(tdb);
  if(tdb->open) tctdbclose(tdb);
  tchdbdel(tdb->hdb);
  if(tdb->mmtx){
    pthread_rwlock_destroy(tdb->mmtx);
    TCFREE(tdb->mmtx);
  }
  TCFREE(tdb);
}
  1. openしてたらクローズ(tctdbclose)
  2. hash dabaseの削除
  3. tdbのmutexを破棄し(pthread.h)、mutexの領域を解放
  4. tdbの領域を解放

[TCTDB]tctdbsetmutex

21:46

/* Set mutual exclusion control of a table database object for threading. */
bool tctdbsetmutex(TCTDB *tdb){
  assert(tdb);
  if(!TCUSEPTHREAD) return true;
  if(tdb->mmtx || tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  TCMALLOC(tdb->mmtx, sizeof(pthread_rwlock_t));
  bool err = false;
  if(pthread_rwlock_init(tdb->mmtx, NULL) != 0) err = true;  if(err){
    TCFREE(tdb->mmtx);
    tdb->mmtx = NULL;
    return false;
  }
  return tchdbsetmutex(tdb->hdb);
}
  1. pthreadを使ってなければなければ何もしない
  2. mutexが既に用意されているか、open済みであればエラー
  3. mutexのヒープ領域を確保し、初期化
  4. hash databaseのmutexを準備
#if defined(_MYNOPTHREAD)
=>efine TCUSEPTHREAD   0
#else
#define TCUSEPTHREAD   1
#endif

[TCTDB]tctdbtune

22:23

/* Set the tuning parameters of a table database object. */
bool tctdbtune(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow, uint8_t opts){
  assert(tdb);
  if(tdb->open){    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  tdb->opts = opts;
  uint8_t hopts = 0;
  if(opts & TDBTLARGE) hopts |= HDBTLARGE;
  if(opts & TDBTDEFLATE) hopts |= HDBTDEFLATE;
  if(opts & TDBTBZIP) hopts |= HDBTBZIP;
  if(opts & TDBTTCBS) hopts |= HDBTTCBS;
  if(opts & TDBTEXCODEC) hopts |= HDBTEXCODEC;
  bnum = (bnum > 0) ? bnum : TDBDEFBNUM;
  apow = (apow >= 0) ? apow : TDBDEFAPOW;
  fpow = (fpow >= 0) ? fpow : TDBDEFFPOW;
  return tchdbtune(tdb->hdb, bnum, apow, fpow, hopts);
}
  1. TDBに設定されているチューニングオプションをhash databaseに設定する
  2. bucket number/alignment power/free block pool powerをhash databaseに設定する
enum {                                   /* enumeration for tuning options */
=>HDBTLARGE = 1 << 0,                    /* use 64-bit bucket array */
  HDBTDEFLATE = 1 << 1,                  /* compress each record with Deflate */
  HDBTBZIP = 1 << 2,                     /* compress each record with BZIP2 */
  HDBTTCBS = 1 << 3,                     /* compress each record with TCBS */
  HDBTEXCODEC = 1 << 4                   /* compress each record with custom functions */
};
=>efine TDBDEFBNUM     131071            // default bucket number                                                                                                            
#define TDBDEFAPOW     4                 // default alignment power                                                                                                          
#define TDBDEFFPOW     10                // default free block pool power 

[TCTDB]tctdbopen

22:57

/* Open a database file and connect a table database object. */
bool tctdbopen(TCTDB *tdb, const char *path, int omode){
  assert(tdb && path);
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdbopenimpl(tdb, path, omode);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}
  1. メソッドロック取得
  2. TDBをopen(tctdbopenimpl)
  3. メソッドロックを解放
/* private macros */
=>efine TDBLOCKMETHOD(TC_tdb, TC_wr)                            \
  ((TC_tdb)->mmtx ? tctdblockmethod((TC_tdb), (TC_wr)) : true)
#define TDBUNLOCKMETHOD(TC_tdb)                         \
  ((TC_tdb)->mmtx ? tctdbunlockmethod(TC_tdb) : true)
#define TDBTHREADYIELD(TC_tdb)                          \
  do { if((TC_tdb)->mmtx) sched_yield(); } while(false)
/* Lock a method of the table database object.                                                                                                                               
   `tdb' specifies the table database object.                                                                                                                                
   `wr' specifies whether the lock is writer or not.                                                                                                                         
   If successful, the return value is true, else, it is false. */
static bool tctdblockmethod(TCTDB *tdb, bool wr){
  assert(tdb);
  if(wr ? pthread_rwlock_wrlock(tdb->mmtx) != 0 : pthread_rwlock_rdlock(tdb->mmtx) != 0){
    tctdbsetecode(tdb, TCETHREAD, __FILE__, __LINE__, __func__);
    return false;
  }
  TCTESTYIELD();
  return true;
}
  1. 引数wrがtrueの場合はwrite/read lockを、falseの場合はread lockを取得
/* Unlock a method of the table database object.                                                                                                                             
   `tdb' specifies the table database object.                                                                                                                                
   If successful, the return value is true, else, it is false. */
static bool tctdbunlockmethod(TCTDB *tdb){
  assert(tdb);
  if(pthread_rwlock_unlock(tdb->mmtx) != 0){
    tctdbsetecode(tdb, TCETHREAD, __FILE__, __LINE__, __func__);
    return false;
  }
  TCTESTYIELD();
  return true;
}
  1. unlockはwrite/read lockを指定してロック解放

[TCTDB]tctdbopenimpl

22:57

/* Open a database file and connect a table database object.                         
   `tdb' specifies the table database object.                                        
   `path' specifies the path of the internal database file.                          
   `omode' specifies the connection mode.                                            
   If successful, the return value is true, else, it is false. */
=>atic bool tctdbopenimpl(TCTDB *tdb, const char *path, int omode){
  assert(tdb && path);
  int dbgfd = tchdbdbgfd(tdb->hdb);
  TCCODEC enc, dec;
  void *encop, *decop;
  tchdbcodecfunc(tdb->hdb, &enc, &encop, &dec, &decop);
  int homode = HDBOREADER;
  int bomode = BDBOREADER;
  if(omode & TDBOWRITER){
    homode = HDBOWRITER;
    bomode = BDBOWRITER;
    if(omode & TDBOCREAT){
      homode |= HDBOCREAT;
      bomode |= BDBOCREAT;
    }
    if(omode & TDBOTRUNC){
      homode |= HDBOTRUNC;
      bomode |= BDBOTRUNC;
    }
    tdb->wmode = true;
  } else {
    tdb->wmode = false;
  }
  if(omode & TDBONOLCK){
    homode |= HDBONOLCK;
    bomode |= BDBONOLCK;
  }
  if(omode & TDBOLCKNB){
    homode |= HDBOLCKNB;
    bomode |= BDBOLCKNB;
  }
  if(omode & TDBOTSYNC){
    homode |= HDBOTSYNC;
    bomode |= BDBOTSYNC;
  }
  tchdbsettype(tdb->hdb, TCDBTTABLE);
  if(!tchdbopen(tdb->hdb, path, homode)) return false;
  char *tpath = tcsprintf("%s%c%s%c*", path, MYEXTCHR, TDBIDXSUFFIX, MYEXTCHR);
  if((omode & TDBOWRITER) && (omode & TDBOTRUNC)){
    TCLIST *paths = tcglobpat(tpath);
    int pnum = TCLISTNUM(paths);
    for(int i = 0; i < pnum; i++){
      unlink(TCLISTVALPTR(paths, i));
    }
    tclistdel(paths);
  }
  TCLIST *paths = tcglobpat(tpath);
  int pnum = TCLISTNUM(paths);
  TCMALLOC(tdb->idxs, sizeof(tdb->idxs[0]) * pnum + 1);
  TDBIDX *idxs = tdb->idxs;
  int inum = 0;
  for(int i = 0; i < pnum; i++){
    const char *ipath = TCLISTVALPTR(paths, i);
    if(!tcstrfwm(ipath, path)) continue;
    const char *rp = ipath + strlen(path);
    if(*rp != MYEXTCHR) continue;
    rp++;
    if(!tcstrfwm(rp, TDBIDXSUFFIX)) continue;
    rp += strlen(TDBIDXSUFFIX);
    if(*rp != MYEXTCHR) continue;
    rp++;
    char *stem = tcstrdup(rp);
    char *ep = strrchr(stem, MYEXTCHR);
    if(!ep) continue;
    *(ep++) = '\0';
    int nsiz;
    char *name = tcurldecode(stem, &nsiz);
    if(!strcmp(ep, "lex") || !strcmp(ep, "dec") || !strcmp(ep, "tok") || !strcmp(ep,\
 "qgr")){
      TCBDB *bdb = tcbdbnew();
      if(dbgfd >= 0) tcbdbsetdbgfd(bdb, dbgfd);
      if(tdb->mmtx) tcbdbsetmutex(bdb);
      if(enc && dec) tcbdbsetcodecfunc(bdb, enc, encop, dec, decop);
      tcbdbsetcache(bdb, tdb->lcnum, tdb->ncnum);
      tcbdbsetxmsiz(bdb, tchdbxmsiz(tdb->hdb));
      tcbdbsetdfunit(bdb, tchdbdfunit(tdb->hdb));
      tcbdbsetlsmax(bdb, TDBIDXLSMAX);
      if(tcbdbopen(bdb, ipath, bomode)){
        idxs[inum].name = tcstrdup(name);
        idxs[inum].type = TDBITLEXICAL;
        if(!strcmp(ep, "dec")){
          idxs[inum].type = TDBITDECIMAL;
        } else if(!strcmp(ep, "tok")){
          idxs[inum].type = TDBITTOKEN;
        } else if(!strcmp(ep, "qgr")){
          idxs[inum].type = TDBITQGRAM;
        }
        idxs[inum].db = bdb;
        idxs[inum].cc = NULL;
        if(idxs[inum].type == TDBITTOKEN){
          idxs[inum].cc = tcmapnew2(TDBIDXICCBNUM);
        } else if(idxs[inum].type == TDBITQGRAM){
          idxs[inum].cc = tcmapnew2(TDBIDXICCBNUM);
        }
        inum++;
      } else {
        tcbdbdel(bdb);
      }
    }
    TCFREE(name);
    TCFREE(stem);
  }
  tclistdel(paths);
  TCFREE(tpath);
  tdb->inum = inum;
  tdb->open = true;
  uint8_t hopts = tchdbopts(tdb->hdb);
  uint8_t opts = 0;
  if(hopts & HDBTLARGE) opts |= TDBTLARGE;
  if(hopts & HDBTDEFLATE) opts |= TDBTDEFLATE;
  if(hopts & HDBTBZIP) opts |= TDBTBZIP;
  if(hopts & HDBTTCBS) opts |= TDBTTCBS;
  if(hopts & HDBTEXCODEC) opts |= TDBTEXCODEC;
  tdb->opts = opts;
  tdb->tran = false;
  return true;
}
  1. tchdbdbgfd(hash databaseのファイルディスクリプタ(デバッグ用))を取得
  2. カスタムコーデック関数を取得
  3. 引数で指定されたomodeによって、hash database、B+tree databaseのモードを決める
  4. hash databaseのtypeにtable databaseセットする
  5. hash databaseをopen
  6. TDBOWRITER 且つ TDBOTRUNCであればインデックスファイルを消す?
  7. インデックス用のB+tree databaseをopen
  8. hash databaseのoptionを取り出して、その一部をtable databaseのoptionとして再設定
/* Get the custom codec functions of a hash database object. */
void tchdbcodecfunc(TCHDB *hdb, TCCODEC *ep, void **eop, TCCODEC *dp, void **dop){
  assert(hdb && ep && eop && dp && dop);
  *ep = hdb->enc;
  *eop = hdb->encop;
  *dp = hdb->dec;
  *dop = hdb->decop;
}
/* type of the pointer to a encoding or decoding function.                           
   `ptr' specifies the pointer to the region.                                        
   `size' specifies the size of the region.                                          
   `sp' specifies the pointer to the variable into which the size of the region of the return                                                                            
   value is assigned.                                                                
   `op' specifies the pointer to the optional opaque object.                         
   If successful, the return value is the pointer to the result object allocated with `malloc'                                                                           
   call, else, it is `NULL'. */
=>pedef void *(*TCCODEC)(const void *ptr, int size, int *sp, void *op);
enum {                                   /* enumeration for database type */
  TCDBTHASH,                             /* hash table */
  TCDBTBTREE,                            /* B+ tree */
  TCDBTFIXED,                            /* fixed-length */=>TCDBTTABLE                             /* table */
};