Hatena::Grouptechmemo

msyktの日記

2012-05-15

[cassandra]RandomAccessReader

02:19

org.apache.cassandra.db.ColumnFamilyStore

    public List<Row> getRangeSlice(ByteBuffer superColumn, final AbstractBounds<RowPosition> range, int maxResults, IFilter columnFilter, List<IndexExpression> rowFilter, boolean maxIsColumns, boolean isPaging)
    {
        return filter(getSequentialIterator(superColumn, range, columnFilter), ExtendedFilter.create(this, columnFilter, rowFilter, maxResults, maxIsColumns, isPaging));
    }

org.apache.cassandra.db.ColumnFamilyStore

    /**
      * Iterate over a range of rows and columns from memtables/sstables.
      *
      * @param superColumn optional SuperColumn to slice subcolumns of; null to slice top-level columns
      * @param range Either a Bounds, which includes start key, or a Range, which does not.
      * @param columnFilter description of the columns we're interested in for each row
     */
    public AbstractScanIterator getSequentialIterator(ByteBuffer superColumn, final AbstractBounds<RowPosition> range, IFilter columnFilter)
    {
        assert !(range instanceof Range) || !((Range)range).isWrapAround() || range.right.isMinimum() : range;

        final RowPosition startWith = range.left;
        final RowPosition stopAt = range.right;

        QueryFilter filter = new QueryFilter(null, new QueryPath(columnFamily, superColumn, null), columnFilter);

        List<Row> rows;
        final ViewFragment view = markReferenced(startWith, stopAt);
        try
        {
            final CloseableIterator<Row> iterator = RowIteratorFactory.getIterator(view.memtables, view.sstables, startWith, stopAt, filter, this);
            final int gcBefore = (int)(System.currentTimeMillis() / 1000) - metadata.getGcGraceSeconds();

...

org.apache.cassandra.db.getIterator


  /**
     * Get a row iterator over the provided memtables and sstables, between the provided keys
     * and filtered by the queryfilter.
     * @param memtables Memtables pending flush.
     * @param sstables SStables to scan through.
     * @param startWith Start at this key
     * @param stopAt Stop and this key
     * @param filter Used to decide which columns to pull out
     * @param cfs
     * @return A row iterator following all the given restrictions
     */
    public static CloseableIterator<Row> getIterator(final Iterable<Memtable> memtables,
                                          final Collection<SSTableReader> sstables,
                                          final RowPosition startWith,
                                          final RowPosition stopAt,
                                          final QueryFilter filter,
                                          final ColumnFamilyStore cfs)
    {
        // fetch data from current memtable, historical memtables, and SSTables in the correct order.
        final List<CloseableIterator<IColumnIterator>> iterators = new ArrayList<CloseableIterator<IColumnIterator>>();

        // memtables
        for (Memtable memtable : memtables)
        {
            iterators.add(new ConvertToColumnIterator(filter, memtable.getEntryIterator(startWith, stopAt)));
        }

        for (SSTableReader sstable : sstables)
        {
            final SSTableScanner scanner = sstable.getScanner(filter);
            scanner.seekTo(startWith);
            iterators.add(scanner);
        }

        // reduce rows from all sources into a single row
        return MergeIterator.get(iterators, COMPARE_BY_KEY, new MergeIterator.Reducer<IColumnIterator, Row>()
        {
            private final int gcBefore = (int) (System.currentTimeMillis() / 1000) - cfs.metadata.getGcGraceSeconds();
            private final List<IColumnIterator> colIters = new ArrayList<IColumnIterator>();
            private DecoratedKey key;
            private ColumnFamily returnCF;

            @Override
            protected void onKeyChange()
            {
                this.returnCF = ColumnFamily.create(cfs.metadata);
            }

            public void reduce(IColumnIterator current)
            {
                this.colIters.add(current);
                this.key = current.getKey();
                this.returnCF.delete(current.getColumnFamily());
            }

            protected Row getReduced()
            {

                // First check if this row is in the rowCache. If it is we can skip the rest
                ColumnFamily cached = cfs.getRawCachedRow(key);
                if (cached == null)
                {
                    // not cached: collate
                    filter.collateColumns(returnCF, colIters, gcBefore);
                }
                else
                {
                    QueryFilter keyFilter = new QueryFilter(key, filter.path, filter.filter);
                    returnCF = cfs.filterColumnFamily(cached, keyFilter, gcBefore);
                }

                Row rv = new Row(key, returnCF);
                colIters.clear();
                key = null;
                return rv;
            }
        });
    }

org.apache.cassandra.io.sstable.SSTableReader

    /**
     *
     * @param filter filter to use when reading the columns
     * @return A Scanner for seeking over the rows of the SSTable.
     */
    public SSTableScanner getScanner(QueryFilter filter)
    {
        return new SSTableScanner(this, filter);
    }

MichaBealleyMichaBealley2017/05/09 07:49For Sale Finasteride Viagra Without Prescription 400 Mg Candian No Script Pharmacy Order Generic Propecia 1mg Generic Viagra 20 Mg <a href=http://byuvaigranonile.com>viagra</a> Pacific Care Pharmacy Port Vila Buy Viagra Online In The Us Viagra Ou Levitra Cialis En Oferta

KennchulkyKennchulky2017/06/15 18:40Cialis Besser Levitra <a href=http://buy-levitra-now.buylevi.com>Buy Levitra Now</a> Prescription Diflucan Acheter Kamagra Gel <a href=http://viacheap.com>buy viagra online</a> Cialis Levitra Comprar Buy Tadacip 20 Mg Cod Pharmacy Online <a href=http://prednisone.ccrpdc.com/buy-prednisone-20mg.php>Buy Prednisone 20mg</a> Propecia Overstock Le Viagra 50 <a href=http://viagra-pills.via100mg.com>Viagra Pills</a> Precios De Cialis

RetDindRetDind2018/02/02 22:26Cialis 20mg Moins Cher Daily Cialis Professional Rx Pharmacy <a href=http://cialviag.com>cialis</a> Cialis De Venta Cialis Mg Dosage

BrceDeattBrceDeatt2018/04/17 07:09the best choice cialis woman the best site cialis tablets
<a href="http://kaivanrosendaal.com/#Buy-Cheap-Cialis-Online">Buy Cheap Cialis in usa</a> how much does a cialis cost get cheap cialis

KatDyedKatDyed2018/04/17 07:28Viagra O Cialis Mejor <a href=http://cialiorder.com>cialis canada</a> Viagra E Dieta

BrceDeattBrceDeatt2018/04/17 23:55cialis kaufen wo cipla cialis online
<a href="http://kaivanrosendaal.com/#Cheap-Cialis">Buy Cheap Cialis in usa</a> 5 mg cialis pharmacie en ligne cialis flussig

BrceDeattBrceDeatt2018/04/18 18:22we use it 50 mg cialis dose brand cialis generic
<a href="http://kaivanrosendaal.com/#Buy-Cheap-Cialis-Online">buy cialis</a> cuanto cuesta cialis yaho if a woman takes a mans cialis

HrafdceDeattHrafdceDeatt2018/04/19 09:13buy 1 cialis pill at a time cialis pakistan price cialis blood in urine <a href=http://jvrimages.com/#cialis+going+generic+in+2017+in+us>cialis cost</a> cialis children cialis generic cheap fast cheapest brand cialis online
<a href=http://jvrimages.com/#cialis+20mg>cialis 30 day sample</a> cialis online uk cialis leggero look there official cialis
http://jvrimages.com/#cialis+daily cialis 88 keys lyrics 110 mg dissolvable cialis cialis e insufficienza renal

AddAcoulaAddAcoula2018/04/19 14:59Pillole Cialis <a href=http://cialviag.com>cialis online</a> Tindamax

HrafdceDeattHrafdceDeatt2018/04/19 22:16shop for cialis cheap buy genuine cialis cialis use by women effective <a href=http://jvrimages.com/#cialis+side+effects+dangers>cialis 20 mg</a> cialis in usa cheap cialis barato profesional cialis plus stay hard
<a href=http://jvrimages.com/#cialis+tadalafil>http://jvrimages.com</a> ipad cialis online sicher buy cialis online howto cialis in mexican pharmacies
http://jvrimages.com/#cialis+5+mg levitra cialis side effects can you take 1 2 cialis the best place cialis buy now

HrafdceDeattHrafdceDeatt2018/04/20 13:10cialis sale internet cialis creaza dependenta cialis deutschland ohne rezept <a href=http://jvrimages.com/#200+cialis+coupon>cialis online</a> acheter cialis site serieux we choice purchase cialis can i buy cialis in malaysia
<a href=http://jvrimages.com/#calluses+on+feet>jvrimages.com/</a> average cialis proce what is the best brand cialis la pillola del cialis
http://jvrimages.com/#30+day+cialis+trial+offer cialis generico venta cialis de cialis baratos cialis posologie

HrafdceDeattHrafdceDeatt2018/04/21 07:13cialis vendo chile cialis pricing in canada orden de cialis barata
п»ї<a href="http://jvrimages.com/">cialis 20 mg</a> buy cheap cialis in usa pharmacy colombo cialis cialis et ses mefaits
cialis online stellen generic cialis to canada cialis erb
http://jvrimages.com/#viagra+or+cialis discount generic cialis uk cialis fedex overnight cialis invention

KufdrgeJearkKufdrgeJeark2018/05/02 17:02we like it cialis price cialis pills in singapore <a href="http://foreigncircles.com/#Trazodone-cheap-tablet">Trazodone cheap tablet</a>
cialis lilly tadalafi cialis pills boards
<a href="http://foreigncircles.com/#Trazodone-cheap-tablet">Trazodone cheap</a>
http://foreigncircles.com

KufdrgeJearkKufdrgeJeark2018/05/02 18:03cialis ahumada i recommend cialis generico <a href="http://foreigncircles.com/#foreigncircles.com">Trazodone cheap</a>
acheter du cialis a geneve weblink price cialis
<a href="http://foreigncircles.com/#Trazodone-cheap-tablet">Trazodone cheap tablet</a>
http://foreigncircles.com

KufdrgeJearkKufdrgeJeark2018/05/02 19:06wow cialis tadalafil 100mg how to purchase cialis on line <a href=http://foreigncircles.com/>foreigncircles.com</a>
cialis per paypa weblink price cialis
<a href=http://foreigncircles.com/#Trazodone-cheap>foreigncircles.com</a>
http://foreigncircles.com

KenneththypeKenneththype2018/05/11 10:38kamagra 100mg chewable tablets
kamagra winkel utrecht
kamagra dosage
kamagra oral jelly uses

RehgeJearkRehgeJeark2018/05/18 07:04<a href="http://humblegaiwan.com/">buy cheap sildenafil online u</a> canadian drugs generic viagra viagra prezzo in linea basso
viagra farmacias guadalajara the best site viagra tablets <a href="http://jardinstea.com/">Buy sildenafil 100 mg for sale</a>
viagra canada on line viagra 5 mg buy
viagra online holland viagra in sconto <a href=http://jardinstea.com/#sildenafil-citrate>sildenafil citrate 100 mg</a>
american pharmacy viagra viagra kamagra levitra
http://humblegaiwan.com/

RehgeJearkRehgeJeark2018/05/18 08:27<a href="http://humblegaiwan.com/">cheapest place to buy sildenafil online</a> when can i take another viagra viagra generico postepay
link for you viagra price achat viagra en europe <a href="http://jardinstea.com/#sildenafil-citrate">sildenafil citrate 100 mg pills</a>
click here take viagra viagra kaufen bankberweisung
viagra kamagra levitra viagra et insomni <a href=http://jardinstea.com/#buy-100-mg>sildenafil 100 mg for sale</a>
acquistare viagra internet viagra 50 mg soft tab
http://humblegaiwan.com/

RehgeJearkRehgeJeark2018/05/18 09:37<a href="http://humblegaiwan.com/">buy sildenafil for cheaper prices</a> viagra 100mg suppliers viagra baratos compran uk
generic viagra 20mg tablets viagra professional from usa <a href="http://humblegaiwan.com/">buy cheap sildenafil online usa</a>
viagra 20mg preis cf buy original viagra
click now viagra from canada buy viagra cheap 10 mg <a href=http://jardinstea.com/#sildenafil>sildenafil 100 mg</a>
achat viagra en europe viagra daily dose generic
http://humblegaiwan.com/

RehgeJearkRehgeJeark2018/05/18 10:50<a href="http://jardinstea.com/#buy-sildenafil">Buy sildenafil 100 mg</a> buying viagra in colombia female viagra no prescription
viagra professional yohimbe where to buy viagra in ontario <a href="http://jardinstea.com/#buy-sildenafil">sildenafil 100 mg</a>
viagra authentique suisse prezzo viagra a buon mercato
generic viagra at the pharmacy viagra professional from usa <a href=http://humblegaiwan.com/>buy sildenafil cheap through online sales</a>
viagra online deutschland viagra 5 mg buy
http://jardinstea.com/#100mg

RehgeJearkRehgeJeark2018/05/18 12:06<a href="http://jardinstea.com/#buy-sildenafil">sildenafil 100 mg coupons</a> venta de viagra canada acheter du viagra a geneve
look here viagra cheap canada generic viagra with dapoxetine <a href="http://jardinstea.com/">sildenafil citrate 100 mg prices</a>
viagra generic tadalafil buy viagra name brand cheap
dose size of viagra viagra for sale south africa <a href=http://humblegaiwan.com/>buy cheap sildenafil thailand</a>
generic viagra pill online viagra en mexico precio
http://jardinstea.com/

RehgeJearkRehgeJeark2018/05/18 13:17<a href="http://humblegaiwan.com/">buy sildenafilcheap</a> acheter viagra meilleur pri click now viagra from canada
does viagra cause gout price viagra best <a href="http://humblegaiwan.com/">sildenafil buy cheap</a>
viagra dose 30mg viagra 20 mg effectiveness
we like it safe cheap viagra preis viagra 20mg schweiz <a href=http://jardinstea.com/>sildenafil 100 mg tablets</a>
the best choice viagra woman viagra pas cher paris
http://jardinstea.com/#buy-sildenafil

RehgeJearkRehgeJeark2018/05/18 14:23<a href="http://humblegaiwan.com/">buy sildenafil for cheaper prices</a> viagra name brand cheap preis viagra 20mg schweiz
viagra 100mg suppliers tarif viagra france <a href="http://jardinstea.com/#buy-sildenafil">sildenafil 100 mg for sale</a>
viagra prezzo in linea basso buy viagra online legal
prices on viagra 10 mg cuanto cuesta viagra yaho <a href=http://jardinstea.com/>sildenafil citrate 100mg coupons</a>
tarif viagra france how much does a viagra cost
http://jardinstea.com/#buy-sildenafil

RehgeJearkRehgeJeark2018/05/18 15:39<a href="http://jardinstea.com/">Buy sildenafil 100 mg</a> we recommend viagra info prices on viagra 10 mg
viagra et insomni viagra canada on line <a href="http://humblegaiwan.com/">buy sildenafil cheapest</a>
buying viagra on internet acheter du viagra a geneve
viagra 100 mg 30 tablet viagra italia gratis <a href=http://humblegaiwan.com/>buy cheap sildenafil 200mg</a>
click here to buy viagra viagra italia gratis
http://humblegaiwan.com/

RehgeJearkRehgeJeark2018/05/18 17:01<a href="http://jardinstea.com/#sildenafil-citrate">sildenafil 100 mg price</a> viagra y deporte prices on viagra 10 mg
are there generic viagra price viagra per pill <a href="http://jardinstea.com/#sildenafil-citrate">Sildenafil Citrate 100 mg best price</a>
buying brand viagra online viagra wir preise
viagra 5 mg buy buy brand viagra cheap <a href=http://jardinstea.com/#buy-100-mg>sildenafil 100 mg tablet cost</a>
safe site to buy viagra online viagra 5 effetti collaterali
http://humblegaiwan.com/

RehgeJearkRehgeJeark2018/05/18 20:21<a href="http://humblegaiwan.com/">buy cheap deal online sildenafil sildenafil</a> viagra e hiv viagra en mexico precio
viagra billig non 5 mg viagra generici <a href="http://jardinstea.com/">Buy sildenafil 100 mg</a>
no prescription viagra cheap acheter viagra kamagra
import viagra viagra generico postepay <a href=http://jardinstea.com/#100mg>sildenafil 100 mg tablets</a>
viagra y deporte viagra 5 mg effetti collateral
http://humblegaiwan.com/

MichaelLophyMichaelLophy2018/06/18 02:40cialis comanda
<a href="http://airvietnamairline.com/">buy cialis</a>
viagra vs cialis vs levitra comparison
<a href="http://airvietnamairline.com/">buy generic cialis online</a>
walmart cialis 20mg price

DanielhizDanielhiz2018/06/21 15:54kamagra 50 mg oral jelly usa
<a href="http://kamagradyn.com/">buy kamagra 100mg</a>
kamagra 100mg oral jelly in india
<a href="http://kamagradyn.com/">buy kamagra 100 mg</a>
kamagra 100mg oral jelly wirkung bei frauen

JaredBumJaredBum2018/06/23 00:48cialis 30 lu tablet 20 mg
<a href="http://cialisprod.com/">buy cialis</a>
when generic cialis coming out
<a href="http://cialisprod.com/">buy cialis</a>
cialis 5mg price comparison

LeroyMutLeroyMut2018/06/24 14:50generic cialis sales
<a href="http://airvietnamairline.com/">buy generic cialis</a>
cialis brand 20 mg
<a href="http://airvietnamairline.com/">buy generic cialis</a>
walmart pharmacy prices cialis

FloydTomFloydTom2018/06/26 19:37kamagra kopen amsterdam
<a href="http://kamagradyn.com/">kamagra 100mg</a>
kamagra 100 gold
<a href="http://kamagraonl.com/">kamagra oral jelly</a>
kamagra uk company

AntonglymnAntonglymn2018/07/04 00:09side effects of prolonged use of cialis
<a href="http://cialisgenericsa.com/">buy cialis</a>
cialis tablets side effects
<a href="http://cialisgenericsa.com/">buy cialis online</a>
cialis 20 mg how quickly does it work

HaroldCaumpHaroldCaump2018/07/13 16:41kamagra 100mg oral jelly suppliers indiana
<a href="https://www.kamagradax.com/">kamagra 100mg</a>
kamagra jelly use
<a href="https://www.kamagradax.com/">kamagra 100mg</a>
kamagra forum pl

CharlesVapCharlesVap2018/07/18 11:39cialis price versus viagra forum
<a href="http://viagrapipls.com/">generic viagra</a>
levitra vs viagra price
<a href="http://viagrapipls.com/">viagra generic</a>
price of generic viagra at cvs
http://viagrapipls.com/
walmart generic viagra 100mg best price

2011-08-13

[TCTDB]tctdbput

22:54

/* Store a record into a table database object. */
bool tctdbput(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){
  assert(tdb && pkbuf && pksiz >= 0 && cols);
  int vsiz;
  if(tcmapget(cols, "", 0, &vsiz)){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;  }
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(!tdb->open || !tdb->wmode){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDOVER);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}
  1. table databaseのロックを取得
  2. tctdbputimpl
  3. table databaseのロックを解放

[TCTDB]tctdbputimpl

22:54

/* Store a record into a table database object.                                                                                                                              
   `tdb' specifies the table database object.                                                                                                                                
   `pkbuf' specifies the pointer to the region of the primary key.                                                                                                           
   `pksiz' specifies the size of the region of the primary key.                                                                                                              
   `cols' specifies a map object containing columns.                                                                                                                         
   `dmode' specifies behavior when the key overlaps.                                                                                                                         
   If successful, the return value is true, else, it is false. */
=>atic bool tctdbputimpl(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols, int dmode){
  assert(tdb && pkbuf && pksiz >= 0 && cols);
  bool err = false;
  int osiz;
  char *obuf = tchdbget(tdb->hdb, pkbuf, pksiz, &osiz);
  if(obuf){
    if(dmode == TDBPDKEEP){
      tctdbsetecode(tdb, TCEKEEP, __FILE__, __LINE__, __func__);
      TCFREE(obuf);
      return false;
    }
    TCMAP *ocols = tcmapload(obuf, osiz);
    if(dmode == TDBPDCAT){
      TCMAP *ncols = tcmapnew2(TCMAPRNUM(cols) + 1);
      tcmapiterinit(cols);
      const char *kbuf;
      int ksiz;
      while((kbuf = tcmapiternext(cols, &ksiz)) != NULL){
        int vsiz;
        const char *vbuf = tcmapiterval(kbuf, &vsiz);
        if(tcmapputkeep(ocols, kbuf, ksiz, vbuf, vsiz)) tcmapput(ncols, kbuf, ksiz, vbuf, vsiz);
      }
      if(!tctdbidxput(tdb, pkbuf, pksiz, ncols)) err = true;
      tcmapdel(ncols);
      int csiz;
      char *cbuf = tcmapdump(ocols, &csiz);
      if(!tchdbput(tdb->hdb, pkbuf, pksiz, cbuf, csiz)) err = true;
      TCFREE(cbuf);
    } else {      
      TCMAP *ncols = tcmapnew2(TCMAPRNUM(cols) + 1);
      tcmapiterinit(cols);
      const char *kbuf;
      int ksiz;
      while((kbuf = tcmapiternext(cols, &ksiz)) != NULL){
        int vsiz;
        const char *vbuf = tcmapiterval(kbuf, &vsiz);
        int osiz;
        const char *obuf = tcmapget(ocols, kbuf, ksiz, &osiz);
        if(obuf && osiz == vsiz && !memcmp(obuf, vbuf, osiz)){
          tcmapout(ocols, kbuf, ksiz);
        } else {
          tcmapput(ncols, kbuf, ksiz, vbuf, vsiz);
        }
      }
      if(!tctdbidxout(tdb, pkbuf, pksiz, ocols)) err = true;
      if(!tctdbidxput(tdb, pkbuf, pksiz, ncols)) err = true;
      tcmapdel(ncols);
      int csiz;      
      char *cbuf = tcmapdump(cols, &csiz);
      if(!tchdbput(tdb->hdb, pkbuf, pksiz, cbuf, csiz)) err = true;
      TCFREE(cbuf);
    }
    tcmapdel(ocols);
    TCFREE(obuf);
  } else {
    if(!tctdbidxput(tdb, pkbuf, pksiz, cols)) err = true;
    int csiz;
    char *cbuf = tcmapdump(cols, &csiz);
    if(!tchdbput(tdb->hdb, pkbuf, pksiz, cbuf, csiz)) err = true;
    TCFREE(cbuf);
  }
  return !err;
}
  1. 指定されたkeyで、hash databaseからエントリを取得
  2. エントリが存在していた場合、以下を実行
    1. dmodeがTDBPDKEEPならエラーで戻る
    2. dmodeがTDBPDCATの場合、以下を実行
      1. 元のmapにcolsの内容を加える
      2. 加えた分のindexを作成
    3. dmodeが(TDBPDKEEP/TDBPDCAT)以外の場合
      1. エントリ内に同一のkeyがあれば当該レコードを削除?(ncolsには登録しない?)
      2. エントリ内に同一のkeyがなければ新しいmapに詰めてhash databaseにput
      3. 旧エントリのindexを削除
      4. 新エントリのindexを作成
  3. エントリが存在していない場合、以下を実行
    1. 指定されたcolsをhash databaseにput

DenverDenver2011/11/29 14:05You really found a way to make this whole prosecs easier.

ruwxiysvruwxiysv2011/11/30 00:36TQ0a7a <a href="http://srfcwslcgnra.com/">srfcwslcgnra</a>

bkyurjuesmsbkyurjuesms2011/12/02 23:56GB3oru <a href="http://mlsskdztafdv.com/">mlsskdztafdv</a>

2011-08-12

[TCTDB]tctdbclose

23:24

/* Close a table database object. */
bool tctdbclose(TCTDB *tdb){
  assert(tdb);
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(!tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdbcloseimpl(tdb);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}
  1. メソッドロック取得
  2. table databaseをclose(tctdbcloseimpl)
  3. メソッドロックを解放

[TCTDB]tctdbcloseimpl

00:00

/* Abort the transaction of a table database object.                                                                                                                         
   `tdb' specifies the table database object.                                                                                                                                
   If successful, the return value is true, else, it is false. */
static bool tctdbtranabortimpl(TCTDB *tdb){
  assert(tdb);
  bool err = false;
  if(!tchdbtranabort(tdb->hdb)) err = true;
  TDBIDX *idxs = tdb->idxs;
  int inum = tdb->inum;
  for(int i = 0; i < inum; i++){
    TDBIDX *idx = idxs + i;
    switch(idx->type){
      case TDBITTOKEN:
      case TDBITQGRAM:
        tcmapclear(idx->cc);
        break;
    }
  }  
  for(int i = 0; i < inum; i++){
    TDBIDX *idx = idxs + i;
    switch(idx->type){
      case TDBITLEXICAL:
      case TDBITDECIMAL:
      case TDBITTOKEN:
      case TDBITQGRAM:
        if(!tcbdbtranabort(idx->db)){
          tctdbsetecode(tdb, tcbdbecode(idx->db), __FILE__, __LINE__, __func__);
          err = true;
        }
        break;
    }
  }
  return !err;
}
  1. hash databaseのtransactionをabort
  2. 各indexに対応するB+tree databaseのtransactionをabort
/* Abort the transaction of a hash database object. */
=>ol tchdbtranabort(TCHDB *hdb){  assert(hdb);
  if(!HDBLOCKMETHOD(hdb, true)) return false;
  if(hdb->fd < 0 || !(hdb->omode & HDBOWRITER) || !hdb->tran){
    tchdbsetecode(hdb, TCEINVALID, __FILE__, __LINE__, __func__);
    HDBUNLOCKMETHOD(hdb);
    return false;
  }
  bool err = false;
  if(hdb->async && !tchdbflushdrp(hdb)) err = true;
  if(!tchdbmemsync(hdb, false)) err = true;
  if(!tchdbwalrestore(hdb, hdb->path)) err = true;
  char hbuf[HDBHEADSIZ];
  if(lseek(hdb->fd, 0, SEEK_SET) == -1){
    tchdbsetecode(hdb, TCESEEK, __FILE__, __LINE__, __func__);
    err = false;
  } else if(!tcread(hdb->fd, hbuf, HDBHEADSIZ)){
    tchdbsetecode(hdb, TCEREAD, __FILE__, __LINE__, __func__);
    err = false;
  } else {
    tchdbloadmeta(hdb, hbuf);
  }
  hdb->dfcur = hdb->frec;
  hdb->iter = 0;
  hdb->xfsiz = 0;
  hdb->fbpnum = 0;
  if(hdb->recc) tcmdbvanish(hdb->recc);
  hdb->tran = false;
  HDBUNLOCKMETHOD(hdb);
  return !err;
}
  1. 非同期であれば、deleyed record poolの内容をmapped memory書き込み
  2. hash database等のメタ情報をmapped memoryに書き込み
  3. Write-Ahead Logging (WAL)によるリストア
  4. メタ情報をmapped memoryから読み直してhash databaseに設定する
/* Abort the transaction of a B+ tree database object. */
=>ol tcbdbtranabort(TCBDB *bdb){  assert(bdb);
  if(!BDBLOCKMETHOD(bdb, true)) return false;
  if(!bdb->open || !bdb->wmode || !bdb->tran){
    tcbdbsetecode(bdb, TCEINVALID, __FILE__, __LINE__, __func__);
    BDBUNLOCKMETHOD(bdb);
    return false;
  }
  tcbdbcachepurge(bdb);
  memcpy(bdb->opaque, bdb->rbopaque, BDBOPAQUESIZ);
  tcbdbloadmeta(bdb);
  TCFREE(bdb->rbopaque);
  bdb->tran = false;
  bdb->rbopaque = NULL;
  bdb->hleaf = 0;
  bdb->lleaf = 0;
  bdb->clock++;
  bool err = false;
  if(!tcbdbcacheadjust(bdb)) err = true;
  if(!tchdbtranvoid(bdb->hdb)) err = true;
  BDBUNLOCKMETHOD(bdb);
  return !err;
}
  1. dirtyなcache pageをクリア
  2. opaqueバッファロールバックopaqueバッファの内容を書き込む
  3. opaqueバッファからメタデータを取り出し、B+ tree databaseに設定する
  4. B+ tree databaseのcache(leaf/node)のアジャスト
  5. 内部のhash databaseのロックを取って、hdb->tran = false

[TCTDB]tchdbmemsync

00:00

/* Synchronize updating contents on memory of a hash database object. */
=>ol tchdbmemsync(TCHDB *hdb, bool phys){  assert(hdb);
  if(hdb->fd < 0 || !(hdb->omode & HDBOWRITER)){
    tchdbsetecode(hdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  bool err = false;
  char hbuf[HDBHEADSIZ];
  tchdbdumpmeta(hdb, hbuf);
  memcpy(hdb->map, hbuf, HDBOPAQUEOFF);
  if(phys){
    size_t xmsiz = (hdb->xmsiz > hdb->msiz) ? hdb->xmsiz : hdb->msiz;
    if(msync(hdb->map, xmsiz, MS_SYNC) == -1){
      tchdbsetecode(hdb, TCEMMAP, __FILE__, __LINE__, __func__);
      err = true;
    }    if(fsync(hdb->fd) == -1){
      tchdbsetecode(hdb, TCESYNC, __FILE__, __LINE__, __func__);
      err = true;
    }
  }
  return !err;
}
  1. hbufにhash databaseのメタ情報(下記コード参照)を書き込む
  2. hash databaseのmapped memoryにメタ情報をコピー
  3. pyhsがtrueの場合、mapped memoryをファイルに同期する
/* Serialize meta data into a buffer.                                                                                                                                        
   `hdb' specifies the hash database object.                                                                                                                                 
   `hbuf' specifies the buffer. */
=>atic void tchdbdumpmeta(TCHDB *hdb, char *hbuf){  memset(hbuf, 0, HDBHEADSIZ);
  sprintf(hbuf, "%s\n%s:%d\n", HDBMAGICDATA, _TC_FORMATVER, _TC_LIBVER);
  memcpy(hbuf + HDBTYPEOFF, &(hdb->type), sizeof(hdb->type));
  memcpy(hbuf + HDBFLAGSOFF, &(hdb->flags), sizeof(hdb->flags));
  memcpy(hbuf + HDBAPOWOFF, &(hdb->apow), sizeof(hdb->apow));
  memcpy(hbuf + HDBFPOWOFF, &(hdb->fpow), sizeof(hdb->fpow));
  memcpy(hbuf + HDBOPTSOFF, &(hdb->opts), sizeof(hdb->opts));
  uint64_t llnum;
  llnum = hdb->bnum;
  llnum = TCHTOILL(llnum);
  memcpy(hbuf + HDBBNUMOFF, &llnum, sizeof(llnum));
  llnum = hdb->rnum;
  llnum = TCHTOILL(llnum);
  memcpy(hbuf + HDBRNUMOFF, &llnum, sizeof(llnum));
  llnum = hdb->fsiz;  llnum = TCHTOILL(llnum);
  memcpy(hbuf + HDBFSIZOFF, &llnum, sizeof(llnum));
  llnum = hdb->frec;
  llnum = TCHTOILL(llnum);
  memcpy(hbuf + HDBFRECOFF, &llnum, sizeof(llnum));
}

2011-08-11TokyoCabinetのTDBのコードを読む

[TCTDB]tctdbnew

21:33

/* Create a table database object. */
TCTDB *tctdbnew(void){
  TCTDB *tdb;
  TCMALLOC(tdb, sizeof(*tdb));
  tctdbclear(tdb);
  tdb->hdb = tchdbnew();
  tchdbtune(tdb->hdb, TDBDEFBNUM, TDBDEFAPOW, TDBDEFFPOW, 0);
  tchdbsetxmsiz(tdb->hdb, TDBDEFXMSIZ);
  return tdb;
}
  1. TDB用のヒープに領域を確保する(TCMALLOCはmalloc)
  2. TDBの各値を初期化
  3. TDBで保持するhash databaseを作成
  4. hash databaseのパラメータを設定
  5. hash databaseのextra mapped memoryのsizeを設定(アラインメントされる)

[TCTDB]tctdbdel

21:46

/* Delete a table database object. */
void tctdbdel(TCTDB *tdb){
  assert(tdb);
  if(tdb->open) tctdbclose(tdb);
  tchdbdel(tdb->hdb);
  if(tdb->mmtx){
    pthread_rwlock_destroy(tdb->mmtx);
    TCFREE(tdb->mmtx);
  }
  TCFREE(tdb);
}
  1. openしてたらクローズ(tctdbclose)
  2. hash dabaseの削除
  3. tdbのmutexを破棄し(pthread.h)、mutexの領域を解放
  4. tdbの領域を解放

[TCTDB]tctdbsetmutex

21:46

/* Set mutual exclusion control of a table database object for threading. */
bool tctdbsetmutex(TCTDB *tdb){
  assert(tdb);
  if(!TCUSEPTHREAD) return true;
  if(tdb->mmtx || tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  TCMALLOC(tdb->mmtx, sizeof(pthread_rwlock_t));
  bool err = false;
  if(pthread_rwlock_init(tdb->mmtx, NULL) != 0) err = true;  if(err){
    TCFREE(tdb->mmtx);
    tdb->mmtx = NULL;
    return false;
  }
  return tchdbsetmutex(tdb->hdb);
}
  1. pthreadを使ってなければなければ何もしない
  2. mutexが既に用意されているか、open済みであればエラー
  3. mutexのヒープ領域を確保し、初期化
  4. hash databaseのmutexを準備
#if defined(_MYNOPTHREAD)
=>efine TCUSEPTHREAD   0
#else
#define TCUSEPTHREAD   1
#endif

[TCTDB]tctdbtune

22:23

/* Set the tuning parameters of a table database object. */
bool tctdbtune(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow, uint8_t opts){
  assert(tdb);
  if(tdb->open){    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  tdb->opts = opts;
  uint8_t hopts = 0;
  if(opts & TDBTLARGE) hopts |= HDBTLARGE;
  if(opts & TDBTDEFLATE) hopts |= HDBTDEFLATE;
  if(opts & TDBTBZIP) hopts |= HDBTBZIP;
  if(opts & TDBTTCBS) hopts |= HDBTTCBS;
  if(opts & TDBTEXCODEC) hopts |= HDBTEXCODEC;
  bnum = (bnum > 0) ? bnum : TDBDEFBNUM;
  apow = (apow >= 0) ? apow : TDBDEFAPOW;
  fpow = (fpow >= 0) ? fpow : TDBDEFFPOW;
  return tchdbtune(tdb->hdb, bnum, apow, fpow, hopts);
}
  1. TDBに設定されているチューニングオプションをhash databaseに設定する
  2. bucket number/alignment power/free block pool powerをhash databaseに設定する
enum {                                   /* enumeration for tuning options */
=>HDBTLARGE = 1 << 0,                    /* use 64-bit bucket array */
  HDBTDEFLATE = 1 << 1,                  /* compress each record with Deflate */
  HDBTBZIP = 1 << 2,                     /* compress each record with BZIP2 */
  HDBTTCBS = 1 << 3,                     /* compress each record with TCBS */
  HDBTEXCODEC = 1 << 4                   /* compress each record with custom functions */
};
=>efine TDBDEFBNUM     131071            // default bucket number                                                                                                            
#define TDBDEFAPOW     4                 // default alignment power                                                                                                          
#define TDBDEFFPOW     10                // default free block pool power 

[TCTDB]tctdbopen

22:57

/* Open a database file and connect a table database object. */
bool tctdbopen(TCTDB *tdb, const char *path, int omode){
  assert(tdb && path);
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdbopenimpl(tdb, path, omode);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}
  1. メソッドロック取得
  2. TDBをopen(tctdbopenimpl)
  3. メソッドロックを解放
/* private macros */
=>efine TDBLOCKMETHOD(TC_tdb, TC_wr)                            \
  ((TC_tdb)->mmtx ? tctdblockmethod((TC_tdb), (TC_wr)) : true)
#define TDBUNLOCKMETHOD(TC_tdb)                         \
  ((TC_tdb)->mmtx ? tctdbunlockmethod(TC_tdb) : true)
#define TDBTHREADYIELD(TC_tdb)                          \
  do { if((TC_tdb)->mmtx) sched_yield(); } while(false)
/* Lock a method of the table database object.                                                                                                                               
   `tdb' specifies the table database object.                                                                                                                                
   `wr' specifies whether the lock is writer or not.                                                                                                                         
   If successful, the return value is true, else, it is false. */
static bool tctdblockmethod(TCTDB *tdb, bool wr){
  assert(tdb);
  if(wr ? pthread_rwlock_wrlock(tdb->mmtx) != 0 : pthread_rwlock_rdlock(tdb->mmtx) != 0){
    tctdbsetecode(tdb, TCETHREAD, __FILE__, __LINE__, __func__);
    return false;
  }
  TCTESTYIELD();
  return true;
}
  1. 引数wrがtrueの場合はwrite/read lockを、falseの場合はread lockを取得
/* Unlock a method of the table database object.                                                                                                                             
   `tdb' specifies the table database object.                                                                                                                                
   If successful, the return value is true, else, it is false. */
static bool tctdbunlockmethod(TCTDB *tdb){
  assert(tdb);
  if(pthread_rwlock_unlock(tdb->mmtx) != 0){
    tctdbsetecode(tdb, TCETHREAD, __FILE__, __LINE__, __func__);
    return false;
  }
  TCTESTYIELD();
  return true;
}
  1. unlockはwrite/read lockを指定してロック解放

[TCTDB]tctdbopenimpl

22:57

/* Open a database file and connect a table database object.                         
   `tdb' specifies the table database object.                                        
   `path' specifies the path of the internal database file.                          
   `omode' specifies the connection mode.                                            
   If successful, the return value is true, else, it is false. */
=>atic bool tctdbopenimpl(TCTDB *tdb, const char *path, int omode){
  assert(tdb && path);
  int dbgfd = tchdbdbgfd(tdb->hdb);
  TCCODEC enc, dec;
  void *encop, *decop;
  tchdbcodecfunc(tdb->hdb, &enc, &encop, &dec, &decop);
  int homode = HDBOREADER;
  int bomode = BDBOREADER;
  if(omode & TDBOWRITER){
    homode = HDBOWRITER;
    bomode = BDBOWRITER;
    if(omode & TDBOCREAT){
      homode |= HDBOCREAT;
      bomode |= BDBOCREAT;
    }
    if(omode & TDBOTRUNC){
      homode |= HDBOTRUNC;
      bomode |= BDBOTRUNC;
    }
    tdb->wmode = true;
  } else {
    tdb->wmode = false;
  }
  if(omode & TDBONOLCK){
    homode |= HDBONOLCK;
    bomode |= BDBONOLCK;
  }
  if(omode & TDBOLCKNB){
    homode |= HDBOLCKNB;
    bomode |= BDBOLCKNB;
  }
  if(omode & TDBOTSYNC){
    homode |= HDBOTSYNC;
    bomode |= BDBOTSYNC;
  }
  tchdbsettype(tdb->hdb, TCDBTTABLE);
  if(!tchdbopen(tdb->hdb, path, homode)) return false;
  char *tpath = tcsprintf("%s%c%s%c*", path, MYEXTCHR, TDBIDXSUFFIX, MYEXTCHR);
  if((omode & TDBOWRITER) && (omode & TDBOTRUNC)){
    TCLIST *paths = tcglobpat(tpath);
    int pnum = TCLISTNUM(paths);
    for(int i = 0; i < pnum; i++){
      unlink(TCLISTVALPTR(paths, i));
    }
    tclistdel(paths);
  }
  TCLIST *paths = tcglobpat(tpath);
  int pnum = TCLISTNUM(paths);
  TCMALLOC(tdb->idxs, sizeof(tdb->idxs[0]) * pnum + 1);
  TDBIDX *idxs = tdb->idxs;
  int inum = 0;
  for(int i = 0; i < pnum; i++){
    const char *ipath = TCLISTVALPTR(paths, i);
    if(!tcstrfwm(ipath, path)) continue;
    const char *rp = ipath + strlen(path);
    if(*rp != MYEXTCHR) continue;
    rp++;
    if(!tcstrfwm(rp, TDBIDXSUFFIX)) continue;
    rp += strlen(TDBIDXSUFFIX);
    if(*rp != MYEXTCHR) continue;
    rp++;
    char *stem = tcstrdup(rp);
    char *ep = strrchr(stem, MYEXTCHR);
    if(!ep) continue;
    *(ep++) = '\0';
    int nsiz;
    char *name = tcurldecode(stem, &nsiz);
    if(!strcmp(ep, "lex") || !strcmp(ep, "dec") || !strcmp(ep, "tok") || !strcmp(ep,\
 "qgr")){
      TCBDB *bdb = tcbdbnew();
      if(dbgfd >= 0) tcbdbsetdbgfd(bdb, dbgfd);
      if(tdb->mmtx) tcbdbsetmutex(bdb);
      if(enc && dec) tcbdbsetcodecfunc(bdb, enc, encop, dec, decop);
      tcbdbsetcache(bdb, tdb->lcnum, tdb->ncnum);
      tcbdbsetxmsiz(bdb, tchdbxmsiz(tdb->hdb));
      tcbdbsetdfunit(bdb, tchdbdfunit(tdb->hdb));
      tcbdbsetlsmax(bdb, TDBIDXLSMAX);
      if(tcbdbopen(bdb, ipath, bomode)){
        idxs[inum].name = tcstrdup(name);
        idxs[inum].type = TDBITLEXICAL;
        if(!strcmp(ep, "dec")){
          idxs[inum].type = TDBITDECIMAL;
        } else if(!strcmp(ep, "tok")){
          idxs[inum].type = TDBITTOKEN;
        } else if(!strcmp(ep, "qgr")){
          idxs[inum].type = TDBITQGRAM;
        }
        idxs[inum].db = bdb;
        idxs[inum].cc = NULL;
        if(idxs[inum].type == TDBITTOKEN){
          idxs[inum].cc = tcmapnew2(TDBIDXICCBNUM);
        } else if(idxs[inum].type == TDBITQGRAM){
          idxs[inum].cc = tcmapnew2(TDBIDXICCBNUM);
        }
        inum++;
      } else {
        tcbdbdel(bdb);
      }
    }
    TCFREE(name);
    TCFREE(stem);
  }
  tclistdel(paths);
  TCFREE(tpath);
  tdb->inum = inum;
  tdb->open = true;
  uint8_t hopts = tchdbopts(tdb->hdb);
  uint8_t opts = 0;
  if(hopts & HDBTLARGE) opts |= TDBTLARGE;
  if(hopts & HDBTDEFLATE) opts |= TDBTDEFLATE;
  if(hopts & HDBTBZIP) opts |= TDBTBZIP;
  if(hopts & HDBTTCBS) opts |= TDBTTCBS;
  if(hopts & HDBTEXCODEC) opts |= TDBTEXCODEC;
  tdb->opts = opts;
  tdb->tran = false;
  return true;
}
  1. tchdbdbgfd(hash databaseのファイルディスクリプタ(デバッグ用))を取得
  2. カスタムコーデック関数を取得
  3. 引数で指定されたomodeによって、hash database、B+tree databaseのモードを決める
  4. hash databaseのtypeにtable databaseセットする
  5. hash databaseをopen
  6. TDBOWRITER 且つ TDBOTRUNCであればインデックスファイルを消す?
  7. インデックス用のB+tree databaseをopen
  8. hash databaseのoptionを取り出して、その一部をtable databaseのoptionとして再設定
/* Get the custom codec functions of a hash database object. */
void tchdbcodecfunc(TCHDB *hdb, TCCODEC *ep, void **eop, TCCODEC *dp, void **dop){
  assert(hdb && ep && eop && dp && dop);
  *ep = hdb->enc;
  *eop = hdb->encop;
  *dp = hdb->dec;
  *dop = hdb->decop;
}
/* type of the pointer to a encoding or decoding function.                           
   `ptr' specifies the pointer to the region.                                        
   `size' specifies the size of the region.                                          
   `sp' specifies the pointer to the variable into which the size of the region of the return                                                                            
   value is assigned.                                                                
   `op' specifies the pointer to the optional opaque object.                         
   If successful, the return value is the pointer to the result object allocated with `malloc'                                                                           
   call, else, it is `NULL'. */
=>pedef void *(*TCCODEC)(const void *ptr, int size, int *sp, void *op);
enum {                                   /* enumeration for database type */
  TCDBTHASH,                             /* hash table */
  TCDBTBTREE,                            /* B+ tree */
  TCDBTFIXED,                            /* fixed-length */=>TCDBTTABLE                             /* table */
};