Annotation of embedaddon/sqlite3/ext/fts1/fts1.c, revision 1.1
1.1 ! misho 1: /* fts1 has a design flaw which can lead to database corruption (see
! 2: ** below). It is recommended not to use it any longer, instead use
! 3: ** fts3 (or higher). If you believe that your use of fts1 is safe,
! 4: ** add -DSQLITE_ENABLE_BROKEN_FTS1=1 to your CFLAGS.
! 5: */
! 6: #if (!defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)) \
! 7: && !defined(SQLITE_ENABLE_BROKEN_FTS1)
! 8: #error fts1 has a design flaw and has been deprecated.
! 9: #endif
! 10: /* The flaw is that fts1 uses the content table's unaliased rowid as
! 11: ** the unique docid. fts1 embeds the rowid in the index it builds,
! 12: ** and expects the rowid to not change. The SQLite VACUUM operation
! 13: ** will renumber such rowids, thereby breaking fts1. If you are using
! 14: ** fts1 in a system which has disabled VACUUM, then you can continue
! 15: ** to use it safely. Note that PRAGMA auto_vacuum does NOT disable
! 16: ** VACUUM, though systems using auto_vacuum are unlikely to invoke
! 17: ** VACUUM.
! 18: **
! 19: ** fts1 should be safe even across VACUUM if you only insert documents
! 20: ** and never delete.
! 21: */
! 22:
! 23: /* The author disclaims copyright to this source code.
! 24: *
! 25: * This is an SQLite module implementing full-text search.
! 26: */
! 27:
! 28: /*
! 29: ** The code in this file is only compiled if:
! 30: **
! 31: ** * The FTS1 module is being built as an extension
! 32: ** (in which case SQLITE_CORE is not defined), or
! 33: **
! 34: ** * The FTS1 module is being built into the core of
! 35: ** SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
! 36: */
! 37: #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
! 38:
! 39: #if defined(SQLITE_ENABLE_FTS1) && !defined(SQLITE_CORE)
! 40: # define SQLITE_CORE 1
! 41: #endif
! 42:
! 43: #include <assert.h>
! 44: #include <stdlib.h>
! 45: #include <stdio.h>
! 46: #include <string.h>
! 47: #include <ctype.h>
! 48:
! 49: #include "fts1.h"
! 50: #include "fts1_hash.h"
! 51: #include "fts1_tokenizer.h"
! 52: #include "sqlite3.h"
! 53: #include "sqlite3ext.h"
! 54: SQLITE_EXTENSION_INIT1
! 55:
! 56:
! 57: #if 0
! 58: # define TRACE(A) printf A; fflush(stdout)
! 59: #else
! 60: # define TRACE(A)
! 61: #endif
! 62:
! 63: /* utility functions */
! 64:
! 65: typedef struct StringBuffer {
! 66: int len; /* length, not including null terminator */
! 67: int alloced; /* Space allocated for s[] */
! 68: char *s; /* Content of the string */
! 69: } StringBuffer;
! 70:
! 71: static void initStringBuffer(StringBuffer *sb){
! 72: sb->len = 0;
! 73: sb->alloced = 100;
! 74: sb->s = malloc(100);
! 75: sb->s[0] = '\0';
! 76: }
! 77:
! 78: static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){
! 79: if( sb->len + nFrom >= sb->alloced ){
! 80: sb->alloced = sb->len + nFrom + 100;
! 81: sb->s = realloc(sb->s, sb->alloced+1);
! 82: if( sb->s==0 ){
! 83: initStringBuffer(sb);
! 84: return;
! 85: }
! 86: }
! 87: memcpy(sb->s + sb->len, zFrom, nFrom);
! 88: sb->len += nFrom;
! 89: sb->s[sb->len] = 0;
! 90: }
! 91: static void append(StringBuffer *sb, const char *zFrom){
! 92: nappend(sb, zFrom, strlen(zFrom));
! 93: }
! 94:
! 95: /* We encode variable-length integers in little-endian order using seven bits
! 96: * per byte as follows:
! 97: **
! 98: ** KEY:
! 99: ** A = 0xxxxxxx 7 bits of data and one flag bit
! 100: ** B = 1xxxxxxx 7 bits of data and one flag bit
! 101: **
! 102: ** 7 bits - A
! 103: ** 14 bits - BA
! 104: ** 21 bits - BBA
! 105: ** and so on.
! 106: */
! 107:
! 108: /* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
! 109: #define VARINT_MAX 10
! 110:
! 111: /* Write a 64-bit variable-length integer to memory starting at p[0].
! 112: * The length of data written will be between 1 and VARINT_MAX bytes.
! 113: * The number of bytes written is returned. */
! 114: static int putVarint(char *p, sqlite_int64 v){
! 115: unsigned char *q = (unsigned char *) p;
! 116: sqlite_uint64 vu = v;
! 117: do{
! 118: *q++ = (unsigned char) ((vu & 0x7f) | 0x80);
! 119: vu >>= 7;
! 120: }while( vu!=0 );
! 121: q[-1] &= 0x7f; /* turn off high bit in final byte */
! 122: assert( q - (unsigned char *)p <= VARINT_MAX );
! 123: return (int) (q - (unsigned char *)p);
! 124: }
! 125:
! 126: /* Read a 64-bit variable-length integer from memory starting at p[0].
! 127: * Return the number of bytes read, or 0 on error.
! 128: * The value is stored in *v. */
! 129: static int getVarint(const char *p, sqlite_int64 *v){
! 130: const unsigned char *q = (const unsigned char *) p;
! 131: sqlite_uint64 x = 0, y = 1;
! 132: while( (*q & 0x80) == 0x80 ){
! 133: x += y * (*q++ & 0x7f);
! 134: y <<= 7;
! 135: if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */
! 136: assert( 0 );
! 137: return 0;
! 138: }
! 139: }
! 140: x += y * (*q++);
! 141: *v = (sqlite_int64) x;
! 142: return (int) (q - (unsigned char *)p);
! 143: }
! 144:
! 145: static int getVarint32(const char *p, int *pi){
! 146: sqlite_int64 i;
! 147: int ret = getVarint(p, &i);
! 148: *pi = (int) i;
! 149: assert( *pi==i );
! 150: return ret;
! 151: }
! 152:
! 153: /*** Document lists ***
! 154: *
! 155: * A document list holds a sorted list of varint-encoded document IDs.
! 156: *
! 157: * A doclist with type DL_POSITIONS_OFFSETS is stored like this:
! 158: *
! 159: * array {
! 160: * varint docid;
! 161: * array {
! 162: * varint position; (delta from previous position plus POS_BASE)
! 163: * varint startOffset; (delta from previous startOffset)
! 164: * varint endOffset; (delta from startOffset)
! 165: * }
! 166: * }
! 167: *
! 168: * Here, array { X } means zero or more occurrences of X, adjacent in memory.
! 169: *
! 170: * A position list may hold positions for text in multiple columns. A position
! 171: * POS_COLUMN is followed by a varint containing the index of the column for
! 172: * following positions in the list. Any positions appearing before any
! 173: * occurrences of POS_COLUMN are for column 0.
! 174: *
! 175: * A doclist with type DL_POSITIONS is like the above, but holds only docids
! 176: * and positions without offset information.
! 177: *
! 178: * A doclist with type DL_DOCIDS is like the above, but holds only docids
! 179: * without positions or offset information.
! 180: *
! 181: * On disk, every document list has positions and offsets, so we don't bother
! 182: * to serialize a doclist's type.
! 183: *
! 184: * We don't yet delta-encode document IDs; doing so will probably be a
! 185: * modest win.
! 186: *
! 187: * NOTE(shess) I've thought of a slightly (1%) better offset encoding.
! 188: * After the first offset, estimate the next offset by using the
! 189: * current token position and the previous token position and offset,
! 190: * offset to handle some variance. So the estimate would be
! 191: * (iPosition*w->iStartOffset/w->iPosition-64), which is delta-encoded
! 192: * as normal. Offsets more than 64 chars from the estimate are
! 193: * encoded as the delta to the previous start offset + 128. An
! 194: * additional tiny increment can be gained by using the end offset of
! 195: * the previous token to make the estimate a tiny bit more precise.
! 196: */
! 197:
! 198: /* It is not safe to call isspace(), tolower(), or isalnum() on
! 199: ** hi-bit-set characters. This is the same solution used in the
! 200: ** tokenizer.
! 201: */
! 202: /* TODO(shess) The snippet-generation code should be using the
! 203: ** tokenizer-generated tokens rather than doing its own local
! 204: ** tokenization.
! 205: */
! 206: /* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
! 207: static int safe_isspace(char c){
! 208: return (c&0x80)==0 ? isspace(c) : 0;
! 209: }
! 210: static int safe_tolower(char c){
! 211: return (c&0x80)==0 ? tolower(c) : c;
! 212: }
! 213: static int safe_isalnum(char c){
! 214: return (c&0x80)==0 ? isalnum(c) : 0;
! 215: }
! 216:
! 217: typedef enum DocListType {
! 218: DL_DOCIDS, /* docids only */
! 219: DL_POSITIONS, /* docids + positions */
! 220: DL_POSITIONS_OFFSETS /* docids + positions + offsets */
! 221: } DocListType;
! 222:
! 223: /*
! 224: ** By default, only positions and not offsets are stored in the doclists.
! 225: ** To change this so that offsets are stored too, compile with
! 226: **
! 227: ** -DDL_DEFAULT=DL_POSITIONS_OFFSETS
! 228: **
! 229: */
! 230: #ifndef DL_DEFAULT
! 231: # define DL_DEFAULT DL_POSITIONS
! 232: #endif
! 233:
! 234: typedef struct DocList {
! 235: char *pData;
! 236: int nData;
! 237: DocListType iType;
! 238: int iLastColumn; /* the last column written */
! 239: int iLastPos; /* the last position written */
! 240: int iLastOffset; /* the last start offset written */
! 241: } DocList;
! 242:
! 243: enum {
! 244: POS_END = 0, /* end of this position list */
! 245: POS_COLUMN, /* followed by new column number */
! 246: POS_BASE
! 247: };
! 248:
! 249: /* Initialize a new DocList to hold the given data. */
! 250: static void docListInit(DocList *d, DocListType iType,
! 251: const char *pData, int nData){
! 252: d->nData = nData;
! 253: if( nData>0 ){
! 254: d->pData = malloc(nData);
! 255: memcpy(d->pData, pData, nData);
! 256: } else {
! 257: d->pData = NULL;
! 258: }
! 259: d->iType = iType;
! 260: d->iLastColumn = 0;
! 261: d->iLastPos = d->iLastOffset = 0;
! 262: }
! 263:
! 264: /* Create a new dynamically-allocated DocList. */
! 265: static DocList *docListNew(DocListType iType){
! 266: DocList *d = (DocList *) malloc(sizeof(DocList));
! 267: docListInit(d, iType, 0, 0);
! 268: return d;
! 269: }
! 270:
! 271: static void docListDestroy(DocList *d){
! 272: free(d->pData);
! 273: #ifndef NDEBUG
! 274: memset(d, 0x55, sizeof(*d));
! 275: #endif
! 276: }
! 277:
! 278: static void docListDelete(DocList *d){
! 279: docListDestroy(d);
! 280: free(d);
! 281: }
! 282:
! 283: static char *docListEnd(DocList *d){
! 284: return d->pData + d->nData;
! 285: }
! 286:
! 287: /* Append a varint to a DocList's data. */
! 288: static void appendVarint(DocList *d, sqlite_int64 i){
! 289: char c[VARINT_MAX];
! 290: int n = putVarint(c, i);
! 291: d->pData = realloc(d->pData, d->nData + n);
! 292: memcpy(d->pData + d->nData, c, n);
! 293: d->nData += n;
! 294: }
! 295:
! 296: static void docListAddDocid(DocList *d, sqlite_int64 iDocid){
! 297: appendVarint(d, iDocid);
! 298: if( d->iType>=DL_POSITIONS ){
! 299: appendVarint(d, POS_END); /* initially empty position list */
! 300: d->iLastColumn = 0;
! 301: d->iLastPos = d->iLastOffset = 0;
! 302: }
! 303: }
! 304:
! 305: /* helper function for docListAddPos and docListAddPosOffset */
! 306: static void addPos(DocList *d, int iColumn, int iPos){
! 307: assert( d->nData>0 );
! 308: --d->nData; /* remove previous terminator */
! 309: if( iColumn!=d->iLastColumn ){
! 310: assert( iColumn>d->iLastColumn );
! 311: appendVarint(d, POS_COLUMN);
! 312: appendVarint(d, iColumn);
! 313: d->iLastColumn = iColumn;
! 314: d->iLastPos = d->iLastOffset = 0;
! 315: }
! 316: assert( iPos>=d->iLastPos );
! 317: appendVarint(d, iPos-d->iLastPos+POS_BASE);
! 318: d->iLastPos = iPos;
! 319: }
! 320:
! 321: /* Add a position to the last position list in a doclist. */
! 322: static void docListAddPos(DocList *d, int iColumn, int iPos){
! 323: assert( d->iType==DL_POSITIONS );
! 324: addPos(d, iColumn, iPos);
! 325: appendVarint(d, POS_END); /* add new terminator */
! 326: }
! 327:
! 328: /*
! 329: ** Add a position and starting and ending offsets to a doclist.
! 330: **
! 331: ** If the doclist is setup to handle only positions, then insert
! 332: ** the position only and ignore the offsets.
! 333: */
! 334: static void docListAddPosOffset(
! 335: DocList *d, /* Doclist under construction */
! 336: int iColumn, /* Column the inserted term is part of */
! 337: int iPos, /* Position of the inserted term */
! 338: int iStartOffset, /* Starting offset of inserted term */
! 339: int iEndOffset /* Ending offset of inserted term */
! 340: ){
! 341: assert( d->iType>=DL_POSITIONS );
! 342: addPos(d, iColumn, iPos);
! 343: if( d->iType==DL_POSITIONS_OFFSETS ){
! 344: assert( iStartOffset>=d->iLastOffset );
! 345: appendVarint(d, iStartOffset-d->iLastOffset);
! 346: d->iLastOffset = iStartOffset;
! 347: assert( iEndOffset>=iStartOffset );
! 348: appendVarint(d, iEndOffset-iStartOffset);
! 349: }
! 350: appendVarint(d, POS_END); /* add new terminator */
! 351: }
! 352:
! 353: /*
! 354: ** A DocListReader object is a cursor into a doclist. Initialize
! 355: ** the cursor to the beginning of the doclist by calling readerInit().
! 356: ** Then use routines
! 357: **
! 358: ** peekDocid()
! 359: ** readDocid()
! 360: ** readPosition()
! 361: ** skipPositionList()
! 362: ** and so forth...
! 363: **
! 364: ** to read information out of the doclist. When we reach the end
! 365: ** of the doclist, atEnd() returns TRUE.
! 366: */
! 367: typedef struct DocListReader {
! 368: DocList *pDoclist; /* The document list we are stepping through */
! 369: char *p; /* Pointer to next unread byte in the doclist */
! 370: int iLastColumn;
! 371: int iLastPos; /* the last position read, or -1 when not in a position list */
! 372: } DocListReader;
! 373:
! 374: /*
! 375: ** Initialize the DocListReader r to point to the beginning of pDoclist.
! 376: */
! 377: static void readerInit(DocListReader *r, DocList *pDoclist){
! 378: r->pDoclist = pDoclist;
! 379: if( pDoclist!=NULL ){
! 380: r->p = pDoclist->pData;
! 381: }
! 382: r->iLastColumn = -1;
! 383: r->iLastPos = -1;
! 384: }
! 385:
! 386: /*
! 387: ** Return TRUE if we have reached then end of pReader and there is
! 388: ** nothing else left to read.
! 389: */
! 390: static int atEnd(DocListReader *pReader){
! 391: return pReader->pDoclist==0 || (pReader->p >= docListEnd(pReader->pDoclist));
! 392: }
! 393:
! 394: /* Peek at the next docid without advancing the read pointer.
! 395: */
! 396: static sqlite_int64 peekDocid(DocListReader *pReader){
! 397: sqlite_int64 ret;
! 398: assert( !atEnd(pReader) );
! 399: assert( pReader->iLastPos==-1 );
! 400: getVarint(pReader->p, &ret);
! 401: return ret;
! 402: }
! 403:
! 404: /* Read the next docid. See also nextDocid().
! 405: */
! 406: static sqlite_int64 readDocid(DocListReader *pReader){
! 407: sqlite_int64 ret;
! 408: assert( !atEnd(pReader) );
! 409: assert( pReader->iLastPos==-1 );
! 410: pReader->p += getVarint(pReader->p, &ret);
! 411: if( pReader->pDoclist->iType>=DL_POSITIONS ){
! 412: pReader->iLastColumn = 0;
! 413: pReader->iLastPos = 0;
! 414: }
! 415: return ret;
! 416: }
! 417:
! 418: /* Read the next position and column index from a position list.
! 419: * Returns the position, or -1 at the end of the list. */
! 420: static int readPosition(DocListReader *pReader, int *iColumn){
! 421: int i;
! 422: int iType = pReader->pDoclist->iType;
! 423:
! 424: if( pReader->iLastPos==-1 ){
! 425: return -1;
! 426: }
! 427: assert( !atEnd(pReader) );
! 428:
! 429: if( iType<DL_POSITIONS ){
! 430: return -1;
! 431: }
! 432: pReader->p += getVarint32(pReader->p, &i);
! 433: if( i==POS_END ){
! 434: pReader->iLastColumn = pReader->iLastPos = -1;
! 435: *iColumn = -1;
! 436: return -1;
! 437: }
! 438: if( i==POS_COLUMN ){
! 439: pReader->p += getVarint32(pReader->p, &pReader->iLastColumn);
! 440: pReader->iLastPos = 0;
! 441: pReader->p += getVarint32(pReader->p, &i);
! 442: assert( i>=POS_BASE );
! 443: }
! 444: pReader->iLastPos += ((int) i)-POS_BASE;
! 445: if( iType>=DL_POSITIONS_OFFSETS ){
! 446: /* Skip over offsets, ignoring them for now. */
! 447: int iStart, iEnd;
! 448: pReader->p += getVarint32(pReader->p, &iStart);
! 449: pReader->p += getVarint32(pReader->p, &iEnd);
! 450: }
! 451: *iColumn = pReader->iLastColumn;
! 452: return pReader->iLastPos;
! 453: }
! 454:
! 455: /* Skip past the end of a position list. */
! 456: static void skipPositionList(DocListReader *pReader){
! 457: DocList *p = pReader->pDoclist;
! 458: if( p && p->iType>=DL_POSITIONS ){
! 459: int iColumn;
! 460: while( readPosition(pReader, &iColumn)!=-1 ){}
! 461: }
! 462: }
! 463:
! 464: /* Skip over a docid, including its position list if the doclist has
! 465: * positions. */
! 466: static void skipDocument(DocListReader *pReader){
! 467: readDocid(pReader);
! 468: skipPositionList(pReader);
! 469: }
! 470:
! 471: /* Skip past all docids which are less than [iDocid]. Returns 1 if a docid
! 472: * matching [iDocid] was found. */
! 473: static int skipToDocid(DocListReader *pReader, sqlite_int64 iDocid){
! 474: sqlite_int64 d = 0;
! 475: while( !atEnd(pReader) && (d=peekDocid(pReader))<iDocid ){
! 476: skipDocument(pReader);
! 477: }
! 478: return !atEnd(pReader) && d==iDocid;
! 479: }
! 480:
! 481: /* Return the first document in a document list.
! 482: */
! 483: static sqlite_int64 firstDocid(DocList *d){
! 484: DocListReader r;
! 485: readerInit(&r, d);
! 486: return readDocid(&r);
! 487: }
! 488:
! 489: #ifdef SQLITE_DEBUG
! 490: /*
! 491: ** This routine is used for debugging purpose only.
! 492: **
! 493: ** Write the content of a doclist to standard output.
! 494: */
! 495: static void printDoclist(DocList *p){
! 496: DocListReader r;
! 497: const char *zSep = "";
! 498:
! 499: readerInit(&r, p);
! 500: while( !atEnd(&r) ){
! 501: sqlite_int64 docid = readDocid(&r);
! 502: if( docid==0 ){
! 503: skipPositionList(&r);
! 504: continue;
! 505: }
! 506: printf("%s%lld", zSep, docid);
! 507: zSep = ",";
! 508: if( p->iType>=DL_POSITIONS ){
! 509: int iPos, iCol;
! 510: const char *zDiv = "";
! 511: printf("(");
! 512: while( (iPos = readPosition(&r, &iCol))>=0 ){
! 513: printf("%s%d:%d", zDiv, iCol, iPos);
! 514: zDiv = ":";
! 515: }
! 516: printf(")");
! 517: }
! 518: }
! 519: printf("\n");
! 520: fflush(stdout);
! 521: }
! 522: #endif /* SQLITE_DEBUG */
! 523:
! 524: /* Trim the given doclist to contain only positions in column
! 525: * [iRestrictColumn]. */
! 526: static void docListRestrictColumn(DocList *in, int iRestrictColumn){
! 527: DocListReader r;
! 528: DocList out;
! 529:
! 530: assert( in->iType>=DL_POSITIONS );
! 531: readerInit(&r, in);
! 532: docListInit(&out, DL_POSITIONS, NULL, 0);
! 533:
! 534: while( !atEnd(&r) ){
! 535: sqlite_int64 iDocid = readDocid(&r);
! 536: int iPos, iColumn;
! 537:
! 538: docListAddDocid(&out, iDocid);
! 539: while( (iPos = readPosition(&r, &iColumn)) != -1 ){
! 540: if( iColumn==iRestrictColumn ){
! 541: docListAddPos(&out, iColumn, iPos);
! 542: }
! 543: }
! 544: }
! 545:
! 546: docListDestroy(in);
! 547: *in = out;
! 548: }
! 549:
! 550: /* Trim the given doclist by discarding any docids without any remaining
! 551: * positions. */
! 552: static void docListDiscardEmpty(DocList *in) {
! 553: DocListReader r;
! 554: DocList out;
! 555:
! 556: /* TODO: It would be nice to implement this operation in place; that
! 557: * could save a significant amount of memory in queries with long doclists. */
! 558: assert( in->iType>=DL_POSITIONS );
! 559: readerInit(&r, in);
! 560: docListInit(&out, DL_POSITIONS, NULL, 0);
! 561:
! 562: while( !atEnd(&r) ){
! 563: sqlite_int64 iDocid = readDocid(&r);
! 564: int match = 0;
! 565: int iPos, iColumn;
! 566: while( (iPos = readPosition(&r, &iColumn)) != -1 ){
! 567: if( !match ){
! 568: docListAddDocid(&out, iDocid);
! 569: match = 1;
! 570: }
! 571: docListAddPos(&out, iColumn, iPos);
! 572: }
! 573: }
! 574:
! 575: docListDestroy(in);
! 576: *in = out;
! 577: }
! 578:
! 579: /* Helper function for docListUpdate() and docListAccumulate().
! 580: ** Splices a doclist element into the doclist represented by r,
! 581: ** leaving r pointing after the newly spliced element.
! 582: */
! 583: static void docListSpliceElement(DocListReader *r, sqlite_int64 iDocid,
! 584: const char *pSource, int nSource){
! 585: DocList *d = r->pDoclist;
! 586: char *pTarget;
! 587: int nTarget, found;
! 588:
! 589: found = skipToDocid(r, iDocid);
! 590:
! 591: /* Describe slice in d to place pSource/nSource. */
! 592: pTarget = r->p;
! 593: if( found ){
! 594: skipDocument(r);
! 595: nTarget = r->p-pTarget;
! 596: }else{
! 597: nTarget = 0;
! 598: }
! 599:
! 600: /* The sense of the following is that there are three possibilities.
! 601: ** If nTarget==nSource, we should not move any memory nor realloc.
! 602: ** If nTarget>nSource, trim target and realloc.
! 603: ** If nTarget<nSource, realloc then expand target.
! 604: */
! 605: if( nTarget>nSource ){
! 606: memmove(pTarget+nSource, pTarget+nTarget, docListEnd(d)-(pTarget+nTarget));
! 607: }
! 608: if( nTarget!=nSource ){
! 609: int iDoclist = pTarget-d->pData;
! 610: d->pData = realloc(d->pData, d->nData+nSource-nTarget);
! 611: pTarget = d->pData+iDoclist;
! 612: }
! 613: if( nTarget<nSource ){
! 614: memmove(pTarget+nSource, pTarget+nTarget, docListEnd(d)-(pTarget+nTarget));
! 615: }
! 616:
! 617: memcpy(pTarget, pSource, nSource);
! 618: d->nData += nSource-nTarget;
! 619: r->p = pTarget+nSource;
! 620: }
! 621:
! 622: /* Insert/update pUpdate into the doclist. */
! 623: static void docListUpdate(DocList *d, DocList *pUpdate){
! 624: DocListReader reader;
! 625:
! 626: assert( d!=NULL && pUpdate!=NULL );
! 627: assert( d->iType==pUpdate->iType);
! 628:
! 629: readerInit(&reader, d);
! 630: docListSpliceElement(&reader, firstDocid(pUpdate),
! 631: pUpdate->pData, pUpdate->nData);
! 632: }
! 633:
! 634: /* Propagate elements from pUpdate to pAcc, overwriting elements with
! 635: ** matching docids.
! 636: */
! 637: static void docListAccumulate(DocList *pAcc, DocList *pUpdate){
! 638: DocListReader accReader, updateReader;
! 639:
! 640: /* Handle edge cases where one doclist is empty. */
! 641: assert( pAcc!=NULL );
! 642: if( pUpdate==NULL || pUpdate->nData==0 ) return;
! 643: if( pAcc->nData==0 ){
! 644: pAcc->pData = malloc(pUpdate->nData);
! 645: memcpy(pAcc->pData, pUpdate->pData, pUpdate->nData);
! 646: pAcc->nData = pUpdate->nData;
! 647: return;
! 648: }
! 649:
! 650: readerInit(&accReader, pAcc);
! 651: readerInit(&updateReader, pUpdate);
! 652:
! 653: while( !atEnd(&updateReader) ){
! 654: char *pSource = updateReader.p;
! 655: sqlite_int64 iDocid = readDocid(&updateReader);
! 656: skipPositionList(&updateReader);
! 657: docListSpliceElement(&accReader, iDocid, pSource, updateReader.p-pSource);
! 658: }
! 659: }
! 660:
! 661: /*
! 662: ** Read the next docid off of pIn. Return 0 if we reach the end.
! 663: *
! 664: * TODO: This assumes that docids are never 0, but they may actually be 0 since
! 665: * users can choose docids when inserting into a full-text table. Fix this.
! 666: */
! 667: static sqlite_int64 nextDocid(DocListReader *pIn){
! 668: skipPositionList(pIn);
! 669: return atEnd(pIn) ? 0 : readDocid(pIn);
! 670: }
! 671:
! 672: /*
! 673: ** pLeft and pRight are two DocListReaders that are pointing to
! 674: ** positions lists of the same document: iDocid.
! 675: **
! 676: ** If there are no instances in pLeft or pRight where the position
! 677: ** of pLeft is one less than the position of pRight, then this
! 678: ** routine adds nothing to pOut.
! 679: **
! 680: ** If there are one or more instances where positions from pLeft
! 681: ** are exactly one less than positions from pRight, then add a new
! 682: ** document record to pOut. If pOut wants to hold positions, then
! 683: ** include the positions from pRight that are one more than a
! 684: ** position in pLeft. In other words: pRight.iPos==pLeft.iPos+1.
! 685: **
! 686: ** pLeft and pRight are left pointing at the next document record.
! 687: */
! 688: static void mergePosList(
! 689: DocListReader *pLeft, /* Left position list */
! 690: DocListReader *pRight, /* Right position list */
! 691: sqlite_int64 iDocid, /* The docid from pLeft and pRight */
! 692: DocList *pOut /* Write the merged document record here */
! 693: ){
! 694: int iLeftCol, iLeftPos = readPosition(pLeft, &iLeftCol);
! 695: int iRightCol, iRightPos = readPosition(pRight, &iRightCol);
! 696: int match = 0;
! 697:
! 698: /* Loop until we've reached the end of both position lists. */
! 699: while( iLeftPos!=-1 && iRightPos!=-1 ){
! 700: if( iLeftCol==iRightCol && iLeftPos+1==iRightPos ){
! 701: if( !match ){
! 702: docListAddDocid(pOut, iDocid);
! 703: match = 1;
! 704: }
! 705: if( pOut->iType>=DL_POSITIONS ){
! 706: docListAddPos(pOut, iRightCol, iRightPos);
! 707: }
! 708: iLeftPos = readPosition(pLeft, &iLeftCol);
! 709: iRightPos = readPosition(pRight, &iRightCol);
! 710: }else if( iRightCol<iLeftCol ||
! 711: (iRightCol==iLeftCol && iRightPos<iLeftPos+1) ){
! 712: iRightPos = readPosition(pRight, &iRightCol);
! 713: }else{
! 714: iLeftPos = readPosition(pLeft, &iLeftCol);
! 715: }
! 716: }
! 717: if( iLeftPos>=0 ) skipPositionList(pLeft);
! 718: if( iRightPos>=0 ) skipPositionList(pRight);
! 719: }
! 720:
! 721: /* We have two doclists: pLeft and pRight.
! 722: ** Write the phrase intersection of these two doclists into pOut.
! 723: **
! 724: ** A phrase intersection means that two documents only match
! 725: ** if pLeft.iPos+1==pRight.iPos.
! 726: **
! 727: ** The output pOut may or may not contain positions. If pOut
! 728: ** does contain positions, they are the positions of pRight.
! 729: */
! 730: static void docListPhraseMerge(
! 731: DocList *pLeft, /* Doclist resulting from the words on the left */
! 732: DocList *pRight, /* Doclist for the next word to the right */
! 733: DocList *pOut /* Write the combined doclist here */
! 734: ){
! 735: DocListReader left, right;
! 736: sqlite_int64 docidLeft, docidRight;
! 737:
! 738: readerInit(&left, pLeft);
! 739: readerInit(&right, pRight);
! 740: docidLeft = nextDocid(&left);
! 741: docidRight = nextDocid(&right);
! 742:
! 743: while( docidLeft>0 && docidRight>0 ){
! 744: if( docidLeft<docidRight ){
! 745: docidLeft = nextDocid(&left);
! 746: }else if( docidRight<docidLeft ){
! 747: docidRight = nextDocid(&right);
! 748: }else{
! 749: mergePosList(&left, &right, docidLeft, pOut);
! 750: docidLeft = nextDocid(&left);
! 751: docidRight = nextDocid(&right);
! 752: }
! 753: }
! 754: }
! 755:
! 756: /* We have two doclists: pLeft and pRight.
! 757: ** Write the intersection of these two doclists into pOut.
! 758: ** Only docids are matched. Position information is ignored.
! 759: **
! 760: ** The output pOut never holds positions.
! 761: */
! 762: static void docListAndMerge(
! 763: DocList *pLeft, /* Doclist resulting from the words on the left */
! 764: DocList *pRight, /* Doclist for the next word to the right */
! 765: DocList *pOut /* Write the combined doclist here */
! 766: ){
! 767: DocListReader left, right;
! 768: sqlite_int64 docidLeft, docidRight;
! 769:
! 770: assert( pOut->iType<DL_POSITIONS );
! 771:
! 772: readerInit(&left, pLeft);
! 773: readerInit(&right, pRight);
! 774: docidLeft = nextDocid(&left);
! 775: docidRight = nextDocid(&right);
! 776:
! 777: while( docidLeft>0 && docidRight>0 ){
! 778: if( docidLeft<docidRight ){
! 779: docidLeft = nextDocid(&left);
! 780: }else if( docidRight<docidLeft ){
! 781: docidRight = nextDocid(&right);
! 782: }else{
! 783: docListAddDocid(pOut, docidLeft);
! 784: docidLeft = nextDocid(&left);
! 785: docidRight = nextDocid(&right);
! 786: }
! 787: }
! 788: }
! 789:
! 790: /* We have two doclists: pLeft and pRight.
! 791: ** Write the union of these two doclists into pOut.
! 792: ** Only docids are matched. Position information is ignored.
! 793: **
! 794: ** The output pOut never holds positions.
! 795: */
! 796: static void docListOrMerge(
! 797: DocList *pLeft, /* Doclist resulting from the words on the left */
! 798: DocList *pRight, /* Doclist for the next word to the right */
! 799: DocList *pOut /* Write the combined doclist here */
! 800: ){
! 801: DocListReader left, right;
! 802: sqlite_int64 docidLeft, docidRight, priorLeft;
! 803:
! 804: readerInit(&left, pLeft);
! 805: readerInit(&right, pRight);
! 806: docidLeft = nextDocid(&left);
! 807: docidRight = nextDocid(&right);
! 808:
! 809: while( docidLeft>0 && docidRight>0 ){
! 810: if( docidLeft<=docidRight ){
! 811: docListAddDocid(pOut, docidLeft);
! 812: }else{
! 813: docListAddDocid(pOut, docidRight);
! 814: }
! 815: priorLeft = docidLeft;
! 816: if( docidLeft<=docidRight ){
! 817: docidLeft = nextDocid(&left);
! 818: }
! 819: if( docidRight>0 && docidRight<=priorLeft ){
! 820: docidRight = nextDocid(&right);
! 821: }
! 822: }
! 823: while( docidLeft>0 ){
! 824: docListAddDocid(pOut, docidLeft);
! 825: docidLeft = nextDocid(&left);
! 826: }
! 827: while( docidRight>0 ){
! 828: docListAddDocid(pOut, docidRight);
! 829: docidRight = nextDocid(&right);
! 830: }
! 831: }
! 832:
! 833: /* We have two doclists: pLeft and pRight.
! 834: ** Write into pOut all documents that occur in pLeft but not
! 835: ** in pRight.
! 836: **
! 837: ** Only docids are matched. Position information is ignored.
! 838: **
! 839: ** The output pOut never holds positions.
! 840: */
! 841: static void docListExceptMerge(
! 842: DocList *pLeft, /* Doclist resulting from the words on the left */
! 843: DocList *pRight, /* Doclist for the next word to the right */
! 844: DocList *pOut /* Write the combined doclist here */
! 845: ){
! 846: DocListReader left, right;
! 847: sqlite_int64 docidLeft, docidRight, priorLeft;
! 848:
! 849: readerInit(&left, pLeft);
! 850: readerInit(&right, pRight);
! 851: docidLeft = nextDocid(&left);
! 852: docidRight = nextDocid(&right);
! 853:
! 854: while( docidLeft>0 && docidRight>0 ){
! 855: priorLeft = docidLeft;
! 856: if( docidLeft<docidRight ){
! 857: docListAddDocid(pOut, docidLeft);
! 858: }
! 859: if( docidLeft<=docidRight ){
! 860: docidLeft = nextDocid(&left);
! 861: }
! 862: if( docidRight>0 && docidRight<=priorLeft ){
! 863: docidRight = nextDocid(&right);
! 864: }
! 865: }
! 866: while( docidLeft>0 ){
! 867: docListAddDocid(pOut, docidLeft);
! 868: docidLeft = nextDocid(&left);
! 869: }
! 870: }
! 871:
! 872: static char *string_dup_n(const char *s, int n){
! 873: char *str = malloc(n + 1);
! 874: memcpy(str, s, n);
! 875: str[n] = '\0';
! 876: return str;
! 877: }
! 878:
! 879: /* Duplicate a string; the caller must free() the returned string.
! 880: * (We don't use strdup() since it is not part of the standard C library and
! 881: * may not be available everywhere.) */
! 882: static char *string_dup(const char *s){
! 883: return string_dup_n(s, strlen(s));
! 884: }
! 885:
! 886: /* Format a string, replacing each occurrence of the % character with
! 887: * zDb.zName. This may be more convenient than sqlite_mprintf()
! 888: * when one string is used repeatedly in a format string.
! 889: * The caller must free() the returned string. */
! 890: static char *string_format(const char *zFormat,
! 891: const char *zDb, const char *zName){
! 892: const char *p;
! 893: size_t len = 0;
! 894: size_t nDb = strlen(zDb);
! 895: size_t nName = strlen(zName);
! 896: size_t nFullTableName = nDb+1+nName;
! 897: char *result;
! 898: char *r;
! 899:
! 900: /* first compute length needed */
! 901: for(p = zFormat ; *p ; ++p){
! 902: len += (*p=='%' ? nFullTableName : 1);
! 903: }
! 904: len += 1; /* for null terminator */
! 905:
! 906: r = result = malloc(len);
! 907: for(p = zFormat; *p; ++p){
! 908: if( *p=='%' ){
! 909: memcpy(r, zDb, nDb);
! 910: r += nDb;
! 911: *r++ = '.';
! 912: memcpy(r, zName, nName);
! 913: r += nName;
! 914: } else {
! 915: *r++ = *p;
! 916: }
! 917: }
! 918: *r++ = '\0';
! 919: assert( r == result + len );
! 920: return result;
! 921: }
! 922:
! 923: static int sql_exec(sqlite3 *db, const char *zDb, const char *zName,
! 924: const char *zFormat){
! 925: char *zCommand = string_format(zFormat, zDb, zName);
! 926: int rc;
! 927: TRACE(("FTS1 sql: %s\n", zCommand));
! 928: rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
! 929: free(zCommand);
! 930: return rc;
! 931: }
! 932:
! 933: static int sql_prepare(sqlite3 *db, const char *zDb, const char *zName,
! 934: sqlite3_stmt **ppStmt, const char *zFormat){
! 935: char *zCommand = string_format(zFormat, zDb, zName);
! 936: int rc;
! 937: TRACE(("FTS1 prepare: %s\n", zCommand));
! 938: rc = sqlite3_prepare(db, zCommand, -1, ppStmt, NULL);
! 939: free(zCommand);
! 940: return rc;
! 941: }
! 942:
! 943: /* end utility functions */
! 944:
! 945: /* Forward reference */
! 946: typedef struct fulltext_vtab fulltext_vtab;
! 947:
! 948: /* A single term in a query is represented by an instances of
! 949: ** the following structure.
! 950: */
! 951: typedef struct QueryTerm {
! 952: short int nPhrase; /* How many following terms are part of the same phrase */
! 953: short int iPhrase; /* This is the i-th term of a phrase. */
! 954: short int iColumn; /* Column of the index that must match this term */
! 955: signed char isOr; /* this term is preceded by "OR" */
! 956: signed char isNot; /* this term is preceded by "-" */
! 957: char *pTerm; /* text of the term. '\000' terminated. malloced */
! 958: int nTerm; /* Number of bytes in pTerm[] */
! 959: } QueryTerm;
! 960:
! 961:
! 962: /* A query string is parsed into a Query structure.
! 963: *
! 964: * We could, in theory, allow query strings to be complicated
! 965: * nested expressions with precedence determined by parentheses.
! 966: * But none of the major search engines do this. (Perhaps the
! 967: * feeling is that an parenthesized expression is two complex of
! 968: * an idea for the average user to grasp.) Taking our lead from
! 969: * the major search engines, we will allow queries to be a list
! 970: * of terms (with an implied AND operator) or phrases in double-quotes,
! 971: * with a single optional "-" before each non-phrase term to designate
! 972: * negation and an optional OR connector.
! 973: *
! 974: * OR binds more tightly than the implied AND, which is what the
! 975: * major search engines seem to do. So, for example:
! 976: *
! 977: * [one two OR three] ==> one AND (two OR three)
! 978: * [one OR two three] ==> (one OR two) AND three
! 979: *
! 980: * A "-" before a term matches all entries that lack that term.
! 981: * The "-" must occur immediately before the term with in intervening
! 982: * space. This is how the search engines do it.
! 983: *
! 984: * A NOT term cannot be the right-hand operand of an OR. If this
! 985: * occurs in the query string, the NOT is ignored:
! 986: *
! 987: * [one OR -two] ==> one OR two
! 988: *
! 989: */
! 990: typedef struct Query {
! 991: fulltext_vtab *pFts; /* The full text index */
! 992: int nTerms; /* Number of terms in the query */
! 993: QueryTerm *pTerms; /* Array of terms. Space obtained from malloc() */
! 994: int nextIsOr; /* Set the isOr flag on the next inserted term */
! 995: int nextColumn; /* Next word parsed must be in this column */
! 996: int dfltColumn; /* The default column */
! 997: } Query;
! 998:
! 999:
! 1000: /*
! 1001: ** An instance of the following structure keeps track of generated
! 1002: ** matching-word offset information and snippets.
! 1003: */
! 1004: typedef struct Snippet {
! 1005: int nMatch; /* Total number of matches */
! 1006: int nAlloc; /* Space allocated for aMatch[] */
! 1007: struct snippetMatch { /* One entry for each matching term */
! 1008: char snStatus; /* Status flag for use while constructing snippets */
! 1009: short int iCol; /* The column that contains the match */
! 1010: short int iTerm; /* The index in Query.pTerms[] of the matching term */
! 1011: short int nByte; /* Number of bytes in the term */
! 1012: int iStart; /* The offset to the first character of the term */
! 1013: } *aMatch; /* Points to space obtained from malloc */
! 1014: char *zOffset; /* Text rendering of aMatch[] */
! 1015: int nOffset; /* strlen(zOffset) */
! 1016: char *zSnippet; /* Snippet text */
! 1017: int nSnippet; /* strlen(zSnippet) */
! 1018: } Snippet;
! 1019:
! 1020:
! 1021: typedef enum QueryType {
! 1022: QUERY_GENERIC, /* table scan */
! 1023: QUERY_ROWID, /* lookup by rowid */
! 1024: QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
! 1025: } QueryType;
! 1026:
! 1027: /* TODO(shess) CHUNK_MAX controls how much data we allow in segment 0
! 1028: ** before we start aggregating into larger segments. Lower CHUNK_MAX
! 1029: ** means that for a given input we have more individual segments per
! 1030: ** term, which means more rows in the table and a bigger index (due to
! 1031: ** both more rows and bigger rowids). But it also reduces the average
! 1032: ** cost of adding new elements to the segment 0 doclist, and it seems
! 1033: ** to reduce the number of pages read and written during inserts. 256
! 1034: ** was chosen by measuring insertion times for a certain input (first
! 1035: ** 10k documents of Enron corpus), though including query performance
! 1036: ** in the decision may argue for a larger value.
! 1037: */
! 1038: #define CHUNK_MAX 256
! 1039:
! 1040: typedef enum fulltext_statement {
! 1041: CONTENT_INSERT_STMT,
! 1042: CONTENT_SELECT_STMT,
! 1043: CONTENT_UPDATE_STMT,
! 1044: CONTENT_DELETE_STMT,
! 1045:
! 1046: TERM_SELECT_STMT,
! 1047: TERM_SELECT_ALL_STMT,
! 1048: TERM_INSERT_STMT,
! 1049: TERM_UPDATE_STMT,
! 1050: TERM_DELETE_STMT,
! 1051:
! 1052: MAX_STMT /* Always at end! */
! 1053: } fulltext_statement;
! 1054:
! 1055: /* These must exactly match the enum above. */
! 1056: /* TODO(adam): Is there some risk that a statement (in particular,
! 1057: ** pTermSelectStmt) will be used in two cursors at once, e.g. if a
! 1058: ** query joins a virtual table to itself? If so perhaps we should
! 1059: ** move some of these to the cursor object.
! 1060: */
! 1061: static const char *const fulltext_zStatement[MAX_STMT] = {
! 1062: /* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */
! 1063: /* CONTENT_SELECT */ "select * from %_content where rowid = ?",
! 1064: /* CONTENT_UPDATE */ NULL, /* generated in contentUpdateStatement() */
! 1065: /* CONTENT_DELETE */ "delete from %_content where rowid = ?",
! 1066:
! 1067: /* TERM_SELECT */
! 1068: "select rowid, doclist from %_term where term = ? and segment = ?",
! 1069: /* TERM_SELECT_ALL */
! 1070: "select doclist from %_term where term = ? order by segment",
! 1071: /* TERM_INSERT */
! 1072: "insert into %_term (rowid, term, segment, doclist) values (?, ?, ?, ?)",
! 1073: /* TERM_UPDATE */ "update %_term set doclist = ? where rowid = ?",
! 1074: /* TERM_DELETE */ "delete from %_term where rowid = ?",
! 1075: };
! 1076:
! 1077: /*
! 1078: ** A connection to a fulltext index is an instance of the following
! 1079: ** structure. The xCreate and xConnect methods create an instance
! 1080: ** of this structure and xDestroy and xDisconnect free that instance.
! 1081: ** All other methods receive a pointer to the structure as one of their
! 1082: ** arguments.
! 1083: */
! 1084: struct fulltext_vtab {
! 1085: sqlite3_vtab base; /* Base class used by SQLite core */
! 1086: sqlite3 *db; /* The database connection */
! 1087: const char *zDb; /* logical database name */
! 1088: const char *zName; /* virtual table name */
! 1089: int nColumn; /* number of columns in virtual table */
! 1090: char **azColumn; /* column names. malloced */
! 1091: char **azContentColumn; /* column names in content table; malloced */
! 1092: sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
! 1093:
! 1094: /* Precompiled statements which we keep as long as the table is
! 1095: ** open.
! 1096: */
! 1097: sqlite3_stmt *pFulltextStatements[MAX_STMT];
! 1098: };
! 1099:
! 1100: /*
! 1101: ** When the core wants to do a query, it create a cursor using a
! 1102: ** call to xOpen. This structure is an instance of a cursor. It
! 1103: ** is destroyed by xClose.
! 1104: */
! 1105: typedef struct fulltext_cursor {
! 1106: sqlite3_vtab_cursor base; /* Base class used by SQLite core */
! 1107: QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */
! 1108: sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
! 1109: int eof; /* True if at End Of Results */
! 1110: Query q; /* Parsed query string */
! 1111: Snippet snippet; /* Cached snippet for the current row */
! 1112: int iColumn; /* Column being searched */
! 1113: DocListReader result; /* used when iCursorType == QUERY_FULLTEXT */
! 1114: } fulltext_cursor;
! 1115:
! 1116: static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){
! 1117: return (fulltext_vtab *) c->base.pVtab;
! 1118: }
! 1119:
! 1120: static const sqlite3_module fulltextModule; /* forward declaration */
! 1121:
! 1122: /* Append a list of strings separated by commas to a StringBuffer. */
! 1123: static void appendList(StringBuffer *sb, int nString, char **azString){
! 1124: int i;
! 1125: for(i=0; i<nString; ++i){
! 1126: if( i>0 ) append(sb, ", ");
! 1127: append(sb, azString[i]);
! 1128: }
! 1129: }
! 1130:
! 1131: /* Return a dynamically generated statement of the form
! 1132: * insert into %_content (rowid, ...) values (?, ...)
! 1133: */
! 1134: static const char *contentInsertStatement(fulltext_vtab *v){
! 1135: StringBuffer sb;
! 1136: int i;
! 1137:
! 1138: initStringBuffer(&sb);
! 1139: append(&sb, "insert into %_content (rowid, ");
! 1140: appendList(&sb, v->nColumn, v->azContentColumn);
! 1141: append(&sb, ") values (?");
! 1142: for(i=0; i<v->nColumn; ++i)
! 1143: append(&sb, ", ?");
! 1144: append(&sb, ")");
! 1145: return sb.s;
! 1146: }
! 1147:
! 1148: /* Return a dynamically generated statement of the form
! 1149: * update %_content set [col_0] = ?, [col_1] = ?, ...
! 1150: * where rowid = ?
! 1151: */
! 1152: static const char *contentUpdateStatement(fulltext_vtab *v){
! 1153: StringBuffer sb;
! 1154: int i;
! 1155:
! 1156: initStringBuffer(&sb);
! 1157: append(&sb, "update %_content set ");
! 1158: for(i=0; i<v->nColumn; ++i) {
! 1159: if( i>0 ){
! 1160: append(&sb, ", ");
! 1161: }
! 1162: append(&sb, v->azContentColumn[i]);
! 1163: append(&sb, " = ?");
! 1164: }
! 1165: append(&sb, " where rowid = ?");
! 1166: return sb.s;
! 1167: }
! 1168:
! 1169: /* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
! 1170: ** If the indicated statement has never been prepared, it is prepared
! 1171: ** and cached, otherwise the cached version is reset.
! 1172: */
! 1173: static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
! 1174: sqlite3_stmt **ppStmt){
! 1175: assert( iStmt<MAX_STMT );
! 1176: if( v->pFulltextStatements[iStmt]==NULL ){
! 1177: const char *zStmt;
! 1178: int rc;
! 1179: switch( iStmt ){
! 1180: case CONTENT_INSERT_STMT:
! 1181: zStmt = contentInsertStatement(v); break;
! 1182: case CONTENT_UPDATE_STMT:
! 1183: zStmt = contentUpdateStatement(v); break;
! 1184: default:
! 1185: zStmt = fulltext_zStatement[iStmt];
! 1186: }
! 1187: rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
! 1188: zStmt);
! 1189: if( zStmt != fulltext_zStatement[iStmt]) free((void *) zStmt);
! 1190: if( rc!=SQLITE_OK ) return rc;
! 1191: } else {
! 1192: int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
! 1193: if( rc!=SQLITE_OK ) return rc;
! 1194: }
! 1195:
! 1196: *ppStmt = v->pFulltextStatements[iStmt];
! 1197: return SQLITE_OK;
! 1198: }
! 1199:
! 1200: /* Step the indicated statement, handling errors SQLITE_BUSY (by
! 1201: ** retrying) and SQLITE_SCHEMA (by re-preparing and transferring
! 1202: ** bindings to the new statement).
! 1203: ** TODO(adam): We should extend this function so that it can work with
! 1204: ** statements declared locally, not only globally cached statements.
! 1205: */
! 1206: static int sql_step_statement(fulltext_vtab *v, fulltext_statement iStmt,
! 1207: sqlite3_stmt **ppStmt){
! 1208: int rc;
! 1209: sqlite3_stmt *s = *ppStmt;
! 1210: assert( iStmt<MAX_STMT );
! 1211: assert( s==v->pFulltextStatements[iStmt] );
! 1212:
! 1213: while( (rc=sqlite3_step(s))!=SQLITE_DONE && rc!=SQLITE_ROW ){
! 1214: if( rc==SQLITE_BUSY ) continue;
! 1215: if( rc!=SQLITE_ERROR ) return rc;
! 1216:
! 1217: /* If an SQLITE_SCHEMA error has occurred, then finalizing this
! 1218: * statement is going to delete the fulltext_vtab structure. If
! 1219: * the statement just executed is in the pFulltextStatements[]
! 1220: * array, it will be finalized twice. So remove it before
! 1221: * calling sqlite3_finalize().
! 1222: */
! 1223: v->pFulltextStatements[iStmt] = NULL;
! 1224: rc = sqlite3_finalize(s);
! 1225: break;
! 1226: }
! 1227: return rc;
! 1228:
! 1229: err:
! 1230: sqlite3_finalize(s);
! 1231: return rc;
! 1232: }
! 1233:
! 1234: /* Like sql_step_statement(), but convert SQLITE_DONE to SQLITE_OK.
! 1235: ** Useful for statements like UPDATE, where we expect no results.
! 1236: */
! 1237: static int sql_single_step_statement(fulltext_vtab *v,
! 1238: fulltext_statement iStmt,
! 1239: sqlite3_stmt **ppStmt){
! 1240: int rc = sql_step_statement(v, iStmt, ppStmt);
! 1241: return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
! 1242: }
! 1243:
! 1244: /* insert into %_content (rowid, ...) values ([rowid], [pValues]) */
! 1245: static int content_insert(fulltext_vtab *v, sqlite3_value *rowid,
! 1246: sqlite3_value **pValues){
! 1247: sqlite3_stmt *s;
! 1248: int i;
! 1249: int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
! 1250: if( rc!=SQLITE_OK ) return rc;
! 1251:
! 1252: rc = sqlite3_bind_value(s, 1, rowid);
! 1253: if( rc!=SQLITE_OK ) return rc;
! 1254:
! 1255: for(i=0; i<v->nColumn; ++i){
! 1256: rc = sqlite3_bind_value(s, 2+i, pValues[i]);
! 1257: if( rc!=SQLITE_OK ) return rc;
! 1258: }
! 1259:
! 1260: return sql_single_step_statement(v, CONTENT_INSERT_STMT, &s);
! 1261: }
! 1262:
! 1263: /* update %_content set col0 = pValues[0], col1 = pValues[1], ...
! 1264: * where rowid = [iRowid] */
! 1265: static int content_update(fulltext_vtab *v, sqlite3_value **pValues,
! 1266: sqlite_int64 iRowid){
! 1267: sqlite3_stmt *s;
! 1268: int i;
! 1269: int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s);
! 1270: if( rc!=SQLITE_OK ) return rc;
! 1271:
! 1272: for(i=0; i<v->nColumn; ++i){
! 1273: rc = sqlite3_bind_value(s, 1+i, pValues[i]);
! 1274: if( rc!=SQLITE_OK ) return rc;
! 1275: }
! 1276:
! 1277: rc = sqlite3_bind_int64(s, 1+v->nColumn, iRowid);
! 1278: if( rc!=SQLITE_OK ) return rc;
! 1279:
! 1280: return sql_single_step_statement(v, CONTENT_UPDATE_STMT, &s);
! 1281: }
! 1282:
! 1283: static void freeStringArray(int nString, const char **pString){
! 1284: int i;
! 1285:
! 1286: for (i=0 ; i < nString ; ++i) {
! 1287: if( pString[i]!=NULL ) free((void *) pString[i]);
! 1288: }
! 1289: free((void *) pString);
! 1290: }
! 1291:
! 1292: /* select * from %_content where rowid = [iRow]
! 1293: * The caller must delete the returned array and all strings in it.
! 1294: * null fields will be NULL in the returned array.
! 1295: *
! 1296: * TODO: Perhaps we should return pointer/length strings here for consistency
! 1297: * with other code which uses pointer/length. */
! 1298: static int content_select(fulltext_vtab *v, sqlite_int64 iRow,
! 1299: const char ***pValues){
! 1300: sqlite3_stmt *s;
! 1301: const char **values;
! 1302: int i;
! 1303: int rc;
! 1304:
! 1305: *pValues = NULL;
! 1306:
! 1307: rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
! 1308: if( rc!=SQLITE_OK ) return rc;
! 1309:
! 1310: rc = sqlite3_bind_int64(s, 1, iRow);
! 1311: if( rc!=SQLITE_OK ) return rc;
! 1312:
! 1313: rc = sql_step_statement(v, CONTENT_SELECT_STMT, &s);
! 1314: if( rc!=SQLITE_ROW ) return rc;
! 1315:
! 1316: values = (const char **) malloc(v->nColumn * sizeof(const char *));
! 1317: for(i=0; i<v->nColumn; ++i){
! 1318: if( sqlite3_column_type(s, i)==SQLITE_NULL ){
! 1319: values[i] = NULL;
! 1320: }else{
! 1321: values[i] = string_dup((char*)sqlite3_column_text(s, i));
! 1322: }
! 1323: }
! 1324:
! 1325: /* We expect only one row. We must execute another sqlite3_step()
! 1326: * to complete the iteration; otherwise the table will remain locked. */
! 1327: rc = sqlite3_step(s);
! 1328: if( rc==SQLITE_DONE ){
! 1329: *pValues = values;
! 1330: return SQLITE_OK;
! 1331: }
! 1332:
! 1333: freeStringArray(v->nColumn, values);
! 1334: return rc;
! 1335: }
! 1336:
! 1337: /* delete from %_content where rowid = [iRow ] */
! 1338: static int content_delete(fulltext_vtab *v, sqlite_int64 iRow){
! 1339: sqlite3_stmt *s;
! 1340: int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
! 1341: if( rc!=SQLITE_OK ) return rc;
! 1342:
! 1343: rc = sqlite3_bind_int64(s, 1, iRow);
! 1344: if( rc!=SQLITE_OK ) return rc;
! 1345:
! 1346: return sql_single_step_statement(v, CONTENT_DELETE_STMT, &s);
! 1347: }
! 1348:
! 1349: /* select rowid, doclist from %_term
! 1350: * where term = [pTerm] and segment = [iSegment]
! 1351: * If found, returns SQLITE_ROW; the caller must free the
! 1352: * returned doclist. If no rows found, returns SQLITE_DONE. */
! 1353: static int term_select(fulltext_vtab *v, const char *pTerm, int nTerm,
! 1354: int iSegment,
! 1355: sqlite_int64 *rowid, DocList *out){
! 1356: sqlite3_stmt *s;
! 1357: int rc = sql_get_statement(v, TERM_SELECT_STMT, &s);
! 1358: if( rc!=SQLITE_OK ) return rc;
! 1359:
! 1360: rc = sqlite3_bind_text(s, 1, pTerm, nTerm, SQLITE_STATIC);
! 1361: if( rc!=SQLITE_OK ) return rc;
! 1362:
! 1363: rc = sqlite3_bind_int(s, 2, iSegment);
! 1364: if( rc!=SQLITE_OK ) return rc;
! 1365:
! 1366: rc = sql_step_statement(v, TERM_SELECT_STMT, &s);
! 1367: if( rc!=SQLITE_ROW ) return rc;
! 1368:
! 1369: *rowid = sqlite3_column_int64(s, 0);
! 1370: docListInit(out, DL_DEFAULT,
! 1371: sqlite3_column_blob(s, 1), sqlite3_column_bytes(s, 1));
! 1372:
! 1373: /* We expect only one row. We must execute another sqlite3_step()
! 1374: * to complete the iteration; otherwise the table will remain locked. */
! 1375: rc = sqlite3_step(s);
! 1376: return rc==SQLITE_DONE ? SQLITE_ROW : rc;
! 1377: }
! 1378:
! 1379: /* Load the segment doclists for term pTerm and merge them in
! 1380: ** appropriate order into out. Returns SQLITE_OK if successful. If
! 1381: ** there are no segments for pTerm, successfully returns an empty
! 1382: ** doclist in out.
! 1383: **
! 1384: ** Each document consists of 1 or more "columns". The number of
! 1385: ** columns is v->nColumn. If iColumn==v->nColumn, then return
! 1386: ** position information about all columns. If iColumn<v->nColumn,
! 1387: ** then only return position information about the iColumn-th column
! 1388: ** (where the first column is 0).
! 1389: */
! 1390: static int term_select_all(
! 1391: fulltext_vtab *v, /* The fulltext index we are querying against */
! 1392: int iColumn, /* If <nColumn, only look at the iColumn-th column */
! 1393: const char *pTerm, /* The term whose posting lists we want */
! 1394: int nTerm, /* Number of bytes in pTerm */
! 1395: DocList *out /* Write the resulting doclist here */
! 1396: ){
! 1397: DocList doclist;
! 1398: sqlite3_stmt *s;
! 1399: int rc = sql_get_statement(v, TERM_SELECT_ALL_STMT, &s);
! 1400: if( rc!=SQLITE_OK ) return rc;
! 1401:
! 1402: rc = sqlite3_bind_text(s, 1, pTerm, nTerm, SQLITE_STATIC);
! 1403: if( rc!=SQLITE_OK ) return rc;
! 1404:
! 1405: docListInit(&doclist, DL_DEFAULT, 0, 0);
! 1406:
! 1407: /* TODO(shess) Handle schema and busy errors. */
! 1408: while( (rc=sql_step_statement(v, TERM_SELECT_ALL_STMT, &s))==SQLITE_ROW ){
! 1409: DocList old;
! 1410:
! 1411: /* TODO(shess) If we processed doclists from oldest to newest, we
! 1412: ** could skip the malloc() involved with the following call. For
! 1413: ** now, I'd rather keep this logic similar to index_insert_term().
! 1414: ** We could additionally drop elements when we see deletes, but
! 1415: ** that would require a distinct version of docListAccumulate().
! 1416: */
! 1417: docListInit(&old, DL_DEFAULT,
! 1418: sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0));
! 1419:
! 1420: if( iColumn<v->nColumn ){ /* querying a single column */
! 1421: docListRestrictColumn(&old, iColumn);
! 1422: }
! 1423:
! 1424: /* doclist contains the newer data, so write it over old. Then
! 1425: ** steal accumulated result for doclist.
! 1426: */
! 1427: docListAccumulate(&old, &doclist);
! 1428: docListDestroy(&doclist);
! 1429: doclist = old;
! 1430: }
! 1431: if( rc!=SQLITE_DONE ){
! 1432: docListDestroy(&doclist);
! 1433: return rc;
! 1434: }
! 1435:
! 1436: docListDiscardEmpty(&doclist);
! 1437: *out = doclist;
! 1438: return SQLITE_OK;
! 1439: }
! 1440:
! 1441: /* insert into %_term (rowid, term, segment, doclist)
! 1442: values ([piRowid], [pTerm], [iSegment], [doclist])
! 1443: ** Lets sqlite select rowid if piRowid is NULL, else uses *piRowid.
! 1444: **
! 1445: ** NOTE(shess) piRowid is IN, with values of "space of int64" plus
! 1446: ** null, it is not used to pass data back to the caller.
! 1447: */
! 1448: static int term_insert(fulltext_vtab *v, sqlite_int64 *piRowid,
! 1449: const char *pTerm, int nTerm,
! 1450: int iSegment, DocList *doclist){
! 1451: sqlite3_stmt *s;
! 1452: int rc = sql_get_statement(v, TERM_INSERT_STMT, &s);
! 1453: if( rc!=SQLITE_OK ) return rc;
! 1454:
! 1455: if( piRowid==NULL ){
! 1456: rc = sqlite3_bind_null(s, 1);
! 1457: }else{
! 1458: rc = sqlite3_bind_int64(s, 1, *piRowid);
! 1459: }
! 1460: if( rc!=SQLITE_OK ) return rc;
! 1461:
! 1462: rc = sqlite3_bind_text(s, 2, pTerm, nTerm, SQLITE_STATIC);
! 1463: if( rc!=SQLITE_OK ) return rc;
! 1464:
! 1465: rc = sqlite3_bind_int(s, 3, iSegment);
! 1466: if( rc!=SQLITE_OK ) return rc;
! 1467:
! 1468: rc = sqlite3_bind_blob(s, 4, doclist->pData, doclist->nData, SQLITE_STATIC);
! 1469: if( rc!=SQLITE_OK ) return rc;
! 1470:
! 1471: return sql_single_step_statement(v, TERM_INSERT_STMT, &s);
! 1472: }
! 1473:
! 1474: /* update %_term set doclist = [doclist] where rowid = [rowid] */
! 1475: static int term_update(fulltext_vtab *v, sqlite_int64 rowid,
! 1476: DocList *doclist){
! 1477: sqlite3_stmt *s;
! 1478: int rc = sql_get_statement(v, TERM_UPDATE_STMT, &s);
! 1479: if( rc!=SQLITE_OK ) return rc;
! 1480:
! 1481: rc = sqlite3_bind_blob(s, 1, doclist->pData, doclist->nData, SQLITE_STATIC);
! 1482: if( rc!=SQLITE_OK ) return rc;
! 1483:
! 1484: rc = sqlite3_bind_int64(s, 2, rowid);
! 1485: if( rc!=SQLITE_OK ) return rc;
! 1486:
! 1487: return sql_single_step_statement(v, TERM_UPDATE_STMT, &s);
! 1488: }
! 1489:
! 1490: static int term_delete(fulltext_vtab *v, sqlite_int64 rowid){
! 1491: sqlite3_stmt *s;
! 1492: int rc = sql_get_statement(v, TERM_DELETE_STMT, &s);
! 1493: if( rc!=SQLITE_OK ) return rc;
! 1494:
! 1495: rc = sqlite3_bind_int64(s, 1, rowid);
! 1496: if( rc!=SQLITE_OK ) return rc;
! 1497:
! 1498: return sql_single_step_statement(v, TERM_DELETE_STMT, &s);
! 1499: }
! 1500:
! 1501: /*
! 1502: ** Free the memory used to contain a fulltext_vtab structure.
! 1503: */
! 1504: static void fulltext_vtab_destroy(fulltext_vtab *v){
! 1505: int iStmt, i;
! 1506:
! 1507: TRACE(("FTS1 Destroy %p\n", v));
! 1508: for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
! 1509: if( v->pFulltextStatements[iStmt]!=NULL ){
! 1510: sqlite3_finalize(v->pFulltextStatements[iStmt]);
! 1511: v->pFulltextStatements[iStmt] = NULL;
! 1512: }
! 1513: }
! 1514:
! 1515: if( v->pTokenizer!=NULL ){
! 1516: v->pTokenizer->pModule->xDestroy(v->pTokenizer);
! 1517: v->pTokenizer = NULL;
! 1518: }
! 1519:
! 1520: free(v->azColumn);
! 1521: for(i = 0; i < v->nColumn; ++i) {
! 1522: sqlite3_free(v->azContentColumn[i]);
! 1523: }
! 1524: free(v->azContentColumn);
! 1525: free(v);
! 1526: }
! 1527:
! 1528: /*
! 1529: ** Token types for parsing the arguments to xConnect or xCreate.
! 1530: */
! 1531: #define TOKEN_EOF 0 /* End of file */
! 1532: #define TOKEN_SPACE 1 /* Any kind of whitespace */
! 1533: #define TOKEN_ID 2 /* An identifier */
! 1534: #define TOKEN_STRING 3 /* A string literal */
! 1535: #define TOKEN_PUNCT 4 /* A single punctuation character */
! 1536:
! 1537: /*
! 1538: ** If X is a character that can be used in an identifier then
! 1539: ** IdChar(X) will be true. Otherwise it is false.
! 1540: **
! 1541: ** For ASCII, any character with the high-order bit set is
! 1542: ** allowed in an identifier. For 7-bit characters,
! 1543: ** sqlite3IsIdChar[X] must be 1.
! 1544: **
! 1545: ** Ticket #1066. the SQL standard does not allow '$' in the
! 1546: ** middle of identfiers. But many SQL implementations do.
! 1547: ** SQLite will allow '$' in identifiers for compatibility.
! 1548: ** But the feature is undocumented.
! 1549: */
! 1550: static const char isIdChar[] = {
! 1551: /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
! 1552: 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
! 1553: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
! 1554: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
! 1555: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
! 1556: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
! 1557: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
! 1558: };
! 1559: #define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isIdChar[c-0x20]))
! 1560:
! 1561:
! 1562: /*
! 1563: ** Return the length of the token that begins at z[0].
! 1564: ** Store the token type in *tokenType before returning.
! 1565: */
! 1566: static int getToken(const char *z, int *tokenType){
! 1567: int i, c;
! 1568: switch( *z ){
! 1569: case 0: {
! 1570: *tokenType = TOKEN_EOF;
! 1571: return 0;
! 1572: }
! 1573: case ' ': case '\t': case '\n': case '\f': case '\r': {
! 1574: for(i=1; safe_isspace(z[i]); i++){}
! 1575: *tokenType = TOKEN_SPACE;
! 1576: return i;
! 1577: }
! 1578: case '`':
! 1579: case '\'':
! 1580: case '"': {
! 1581: int delim = z[0];
! 1582: for(i=1; (c=z[i])!=0; i++){
! 1583: if( c==delim ){
! 1584: if( z[i+1]==delim ){
! 1585: i++;
! 1586: }else{
! 1587: break;
! 1588: }
! 1589: }
! 1590: }
! 1591: *tokenType = TOKEN_STRING;
! 1592: return i + (c!=0);
! 1593: }
! 1594: case '[': {
! 1595: for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
! 1596: *tokenType = TOKEN_ID;
! 1597: return i;
! 1598: }
! 1599: default: {
! 1600: if( !IdChar(*z) ){
! 1601: break;
! 1602: }
! 1603: for(i=1; IdChar(z[i]); i++){}
! 1604: *tokenType = TOKEN_ID;
! 1605: return i;
! 1606: }
! 1607: }
! 1608: *tokenType = TOKEN_PUNCT;
! 1609: return 1;
! 1610: }
! 1611:
! 1612: /*
! 1613: ** A token extracted from a string is an instance of the following
! 1614: ** structure.
! 1615: */
! 1616: typedef struct Token {
! 1617: const char *z; /* Pointer to token text. Not '\000' terminated */
! 1618: short int n; /* Length of the token text in bytes. */
! 1619: } Token;
! 1620:
! 1621: /*
! 1622: ** Given a input string (which is really one of the argv[] parameters
! 1623: ** passed into xConnect or xCreate) split the string up into tokens.
! 1624: ** Return an array of pointers to '\000' terminated strings, one string
! 1625: ** for each non-whitespace token.
! 1626: **
! 1627: ** The returned array is terminated by a single NULL pointer.
! 1628: **
! 1629: ** Space to hold the returned array is obtained from a single
! 1630: ** malloc and should be freed by passing the return value to free().
! 1631: ** The individual strings within the token list are all a part of
! 1632: ** the single memory allocation and will all be freed at once.
! 1633: */
! 1634: static char **tokenizeString(const char *z, int *pnToken){
! 1635: int nToken = 0;
! 1636: Token *aToken = malloc( strlen(z) * sizeof(aToken[0]) );
! 1637: int n = 1;
! 1638: int e, i;
! 1639: int totalSize = 0;
! 1640: char **azToken;
! 1641: char *zCopy;
! 1642: while( n>0 ){
! 1643: n = getToken(z, &e);
! 1644: if( e!=TOKEN_SPACE ){
! 1645: aToken[nToken].z = z;
! 1646: aToken[nToken].n = n;
! 1647: nToken++;
! 1648: totalSize += n+1;
! 1649: }
! 1650: z += n;
! 1651: }
! 1652: azToken = (char**)malloc( nToken*sizeof(char*) + totalSize );
! 1653: zCopy = (char*)&azToken[nToken];
! 1654: nToken--;
! 1655: for(i=0; i<nToken; i++){
! 1656: azToken[i] = zCopy;
! 1657: n = aToken[i].n;
! 1658: memcpy(zCopy, aToken[i].z, n);
! 1659: zCopy[n] = 0;
! 1660: zCopy += n+1;
! 1661: }
! 1662: azToken[nToken] = 0;
! 1663: free(aToken);
! 1664: *pnToken = nToken;
! 1665: return azToken;
! 1666: }
! 1667:
! 1668: /*
! 1669: ** Convert an SQL-style quoted string into a normal string by removing
! 1670: ** the quote characters. The conversion is done in-place. If the
! 1671: ** input does not begin with a quote character, then this routine
! 1672: ** is a no-op.
! 1673: **
! 1674: ** Examples:
! 1675: **
! 1676: ** "abc" becomes abc
! 1677: ** 'xyz' becomes xyz
! 1678: ** [pqr] becomes pqr
! 1679: ** `mno` becomes mno
! 1680: */
! 1681: static void dequoteString(char *z){
! 1682: int quote;
! 1683: int i, j;
! 1684: if( z==0 ) return;
! 1685: quote = z[0];
! 1686: switch( quote ){
! 1687: case '\'': break;
! 1688: case '"': break;
! 1689: case '`': break; /* For MySQL compatibility */
! 1690: case '[': quote = ']'; break; /* For MS SqlServer compatibility */
! 1691: default: return;
! 1692: }
! 1693: for(i=1, j=0; z[i]; i++){
! 1694: if( z[i]==quote ){
! 1695: if( z[i+1]==quote ){
! 1696: z[j++] = quote;
! 1697: i++;
! 1698: }else{
! 1699: z[j++] = 0;
! 1700: break;
! 1701: }
! 1702: }else{
! 1703: z[j++] = z[i];
! 1704: }
! 1705: }
! 1706: }
! 1707:
! 1708: /*
! 1709: ** The input azIn is a NULL-terminated list of tokens. Remove the first
! 1710: ** token and all punctuation tokens. Remove the quotes from
! 1711: ** around string literal tokens.
! 1712: **
! 1713: ** Example:
! 1714: **
! 1715: ** input: tokenize chinese ( 'simplifed' , 'mixed' )
! 1716: ** output: chinese simplifed mixed
! 1717: **
! 1718: ** Another example:
! 1719: **
! 1720: ** input: delimiters ( '[' , ']' , '...' )
! 1721: ** output: [ ] ...
! 1722: */
! 1723: static void tokenListToIdList(char **azIn){
! 1724: int i, j;
! 1725: if( azIn ){
! 1726: for(i=0, j=-1; azIn[i]; i++){
! 1727: if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){
! 1728: dequoteString(azIn[i]);
! 1729: if( j>=0 ){
! 1730: azIn[j] = azIn[i];
! 1731: }
! 1732: j++;
! 1733: }
! 1734: }
! 1735: azIn[j] = 0;
! 1736: }
! 1737: }
! 1738:
! 1739:
! 1740: /*
! 1741: ** Find the first alphanumeric token in the string zIn. Null-terminate
! 1742: ** this token. Remove any quotation marks. And return a pointer to
! 1743: ** the result.
! 1744: */
! 1745: static char *firstToken(char *zIn, char **pzTail){
! 1746: int n, ttype;
! 1747: while(1){
! 1748: n = getToken(zIn, &ttype);
! 1749: if( ttype==TOKEN_SPACE ){
! 1750: zIn += n;
! 1751: }else if( ttype==TOKEN_EOF ){
! 1752: *pzTail = zIn;
! 1753: return 0;
! 1754: }else{
! 1755: zIn[n] = 0;
! 1756: *pzTail = &zIn[1];
! 1757: dequoteString(zIn);
! 1758: return zIn;
! 1759: }
! 1760: }
! 1761: /*NOTREACHED*/
! 1762: }
! 1763:
! 1764: /* Return true if...
! 1765: **
! 1766: ** * s begins with the string t, ignoring case
! 1767: ** * s is longer than t
! 1768: ** * The first character of s beyond t is not a alphanumeric
! 1769: **
! 1770: ** Ignore leading space in *s.
! 1771: **
! 1772: ** To put it another way, return true if the first token of
! 1773: ** s[] is t[].
! 1774: */
! 1775: static int startsWith(const char *s, const char *t){
! 1776: while( safe_isspace(*s) ){ s++; }
! 1777: while( *t ){
! 1778: if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0;
! 1779: }
! 1780: return *s!='_' && !safe_isalnum(*s);
! 1781: }
! 1782:
! 1783: /*
! 1784: ** An instance of this structure defines the "spec" of a
! 1785: ** full text index. This structure is populated by parseSpec
! 1786: ** and use by fulltextConnect and fulltextCreate.
! 1787: */
! 1788: typedef struct TableSpec {
! 1789: const char *zDb; /* Logical database name */
! 1790: const char *zName; /* Name of the full-text index */
! 1791: int nColumn; /* Number of columns to be indexed */
! 1792: char **azColumn; /* Original names of columns to be indexed */
! 1793: char **azContentColumn; /* Column names for %_content */
! 1794: char **azTokenizer; /* Name of tokenizer and its arguments */
! 1795: } TableSpec;
! 1796:
! 1797: /*
! 1798: ** Reclaim all of the memory used by a TableSpec
! 1799: */
! 1800: static void clearTableSpec(TableSpec *p) {
! 1801: free(p->azColumn);
! 1802: free(p->azContentColumn);
! 1803: free(p->azTokenizer);
! 1804: }
! 1805:
! 1806: /* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
! 1807: *
! 1808: * CREATE VIRTUAL TABLE email
! 1809: * USING fts1(subject, body, tokenize mytokenizer(myarg))
! 1810: *
! 1811: * We return parsed information in a TableSpec structure.
! 1812: *
! 1813: */
! 1814: static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv,
! 1815: char**pzErr){
! 1816: int i, n;
! 1817: char *z, *zDummy;
! 1818: char **azArg;
! 1819: const char *zTokenizer = 0; /* argv[] entry describing the tokenizer */
! 1820:
! 1821: assert( argc>=3 );
! 1822: /* Current interface:
! 1823: ** argv[0] - module name
! 1824: ** argv[1] - database name
! 1825: ** argv[2] - table name
! 1826: ** argv[3..] - columns, optionally followed by tokenizer specification
! 1827: ** and snippet delimiters specification.
! 1828: */
! 1829:
! 1830: /* Make a copy of the complete argv[][] array in a single allocation.
! 1831: ** The argv[][] array is read-only and transient. We can write to the
! 1832: ** copy in order to modify things and the copy is persistent.
! 1833: */
! 1834: memset(pSpec, 0, sizeof(*pSpec));
! 1835: for(i=n=0; i<argc; i++){
! 1836: n += strlen(argv[i]) + 1;
! 1837: }
! 1838: azArg = malloc( sizeof(char*)*argc + n );
! 1839: if( azArg==0 ){
! 1840: return SQLITE_NOMEM;
! 1841: }
! 1842: z = (char*)&azArg[argc];
! 1843: for(i=0; i<argc; i++){
! 1844: azArg[i] = z;
! 1845: strcpy(z, argv[i]);
! 1846: z += strlen(z)+1;
! 1847: }
! 1848:
! 1849: /* Identify the column names and the tokenizer and delimiter arguments
! 1850: ** in the argv[][] array.
! 1851: */
! 1852: pSpec->zDb = azArg[1];
! 1853: pSpec->zName = azArg[2];
! 1854: pSpec->nColumn = 0;
! 1855: pSpec->azColumn = azArg;
! 1856: zTokenizer = "tokenize simple";
! 1857: for(i=3; i<argc; ++i){
! 1858: if( startsWith(azArg[i],"tokenize") ){
! 1859: zTokenizer = azArg[i];
! 1860: }else{
! 1861: z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
! 1862: pSpec->nColumn++;
! 1863: }
! 1864: }
! 1865: if( pSpec->nColumn==0 ){
! 1866: azArg[0] = "content";
! 1867: pSpec->nColumn = 1;
! 1868: }
! 1869:
! 1870: /*
! 1871: ** Construct the list of content column names.
! 1872: **
! 1873: ** Each content column name will be of the form cNNAAAA
! 1874: ** where NN is the column number and AAAA is the sanitized
! 1875: ** column name. "sanitized" means that special characters are
! 1876: ** converted to "_". The cNN prefix guarantees that all column
! 1877: ** names are unique.
! 1878: **
! 1879: ** The AAAA suffix is not strictly necessary. It is included
! 1880: ** for the convenience of people who might examine the generated
! 1881: ** %_content table and wonder what the columns are used for.
! 1882: */
! 1883: pSpec->azContentColumn = malloc( pSpec->nColumn * sizeof(char *) );
! 1884: if( pSpec->azContentColumn==0 ){
! 1885: clearTableSpec(pSpec);
! 1886: return SQLITE_NOMEM;
! 1887: }
! 1888: for(i=0; i<pSpec->nColumn; i++){
! 1889: char *p;
! 1890: pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
! 1891: for (p = pSpec->azContentColumn[i]; *p ; ++p) {
! 1892: if( !safe_isalnum(*p) ) *p = '_';
! 1893: }
! 1894: }
! 1895:
! 1896: /*
! 1897: ** Parse the tokenizer specification string.
! 1898: */
! 1899: pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
! 1900: tokenListToIdList(pSpec->azTokenizer);
! 1901:
! 1902: return SQLITE_OK;
! 1903: }
! 1904:
! 1905: /*
! 1906: ** Generate a CREATE TABLE statement that describes the schema of
! 1907: ** the virtual table. Return a pointer to this schema string.
! 1908: **
! 1909: ** Space is obtained from sqlite3_mprintf() and should be freed
! 1910: ** using sqlite3_free().
! 1911: */
! 1912: static char *fulltextSchema(
! 1913: int nColumn, /* Number of columns */
! 1914: const char *const* azColumn, /* List of columns */
! 1915: const char *zTableName /* Name of the table */
! 1916: ){
! 1917: int i;
! 1918: char *zSchema, *zNext;
! 1919: const char *zSep = "(";
! 1920: zSchema = sqlite3_mprintf("CREATE TABLE x");
! 1921: for(i=0; i<nColumn; i++){
! 1922: zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
! 1923: sqlite3_free(zSchema);
! 1924: zSchema = zNext;
! 1925: zSep = ",";
! 1926: }
! 1927: zNext = sqlite3_mprintf("%s,%Q)", zSchema, zTableName);
! 1928: sqlite3_free(zSchema);
! 1929: return zNext;
! 1930: }
! 1931:
! 1932: /*
! 1933: ** Build a new sqlite3_vtab structure that will describe the
! 1934: ** fulltext index defined by spec.
! 1935: */
! 1936: static int constructVtab(
! 1937: sqlite3 *db, /* The SQLite database connection */
! 1938: TableSpec *spec, /* Parsed spec information from parseSpec() */
! 1939: sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
! 1940: char **pzErr /* Write any error message here */
! 1941: ){
! 1942: int rc;
! 1943: int n;
! 1944: fulltext_vtab *v = 0;
! 1945: const sqlite3_tokenizer_module *m = NULL;
! 1946: char *schema;
! 1947:
! 1948: v = (fulltext_vtab *) malloc(sizeof(fulltext_vtab));
! 1949: if( v==0 ) return SQLITE_NOMEM;
! 1950: memset(v, 0, sizeof(*v));
! 1951: /* sqlite will initialize v->base */
! 1952: v->db = db;
! 1953: v->zDb = spec->zDb; /* Freed when azColumn is freed */
! 1954: v->zName = spec->zName; /* Freed when azColumn is freed */
! 1955: v->nColumn = spec->nColumn;
! 1956: v->azContentColumn = spec->azContentColumn;
! 1957: spec->azContentColumn = 0;
! 1958: v->azColumn = spec->azColumn;
! 1959: spec->azColumn = 0;
! 1960:
! 1961: if( spec->azTokenizer==0 ){
! 1962: return SQLITE_NOMEM;
! 1963: }
! 1964: /* TODO(shess) For now, add new tokenizers as else if clauses. */
! 1965: if( spec->azTokenizer[0]==0 || startsWith(spec->azTokenizer[0], "simple") ){
! 1966: sqlite3Fts1SimpleTokenizerModule(&m);
! 1967: }else if( startsWith(spec->azTokenizer[0], "porter") ){
! 1968: sqlite3Fts1PorterTokenizerModule(&m);
! 1969: }else{
! 1970: *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
! 1971: rc = SQLITE_ERROR;
! 1972: goto err;
! 1973: }
! 1974: for(n=0; spec->azTokenizer[n]; n++){}
! 1975: if( n ){
! 1976: rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1],
! 1977: &v->pTokenizer);
! 1978: }else{
! 1979: rc = m->xCreate(0, 0, &v->pTokenizer);
! 1980: }
! 1981: if( rc!=SQLITE_OK ) goto err;
! 1982: v->pTokenizer->pModule = m;
! 1983:
! 1984: /* TODO: verify the existence of backing tables foo_content, foo_term */
! 1985:
! 1986: schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn,
! 1987: spec->zName);
! 1988: rc = sqlite3_declare_vtab(db, schema);
! 1989: sqlite3_free(schema);
! 1990: if( rc!=SQLITE_OK ) goto err;
! 1991:
! 1992: memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
! 1993:
! 1994: *ppVTab = &v->base;
! 1995: TRACE(("FTS1 Connect %p\n", v));
! 1996:
! 1997: return rc;
! 1998:
! 1999: err:
! 2000: fulltext_vtab_destroy(v);
! 2001: return rc;
! 2002: }
! 2003:
! 2004: static int fulltextConnect(
! 2005: sqlite3 *db,
! 2006: void *pAux,
! 2007: int argc, const char *const*argv,
! 2008: sqlite3_vtab **ppVTab,
! 2009: char **pzErr
! 2010: ){
! 2011: TableSpec spec;
! 2012: int rc = parseSpec(&spec, argc, argv, pzErr);
! 2013: if( rc!=SQLITE_OK ) return rc;
! 2014:
! 2015: rc = constructVtab(db, &spec, ppVTab, pzErr);
! 2016: clearTableSpec(&spec);
! 2017: return rc;
! 2018: }
! 2019:
! 2020: /* The %_content table holds the text of each document, with
! 2021: ** the rowid used as the docid.
! 2022: **
! 2023: ** The %_term table maps each term to a document list blob
! 2024: ** containing elements sorted by ascending docid, each element
! 2025: ** encoded as:
! 2026: **
! 2027: ** docid varint-encoded
! 2028: ** token elements:
! 2029: ** position+1 varint-encoded as delta from previous position
! 2030: ** start offset varint-encoded as delta from previous start offset
! 2031: ** end offset varint-encoded as delta from start offset
! 2032: **
! 2033: ** The sentinel position of 0 indicates the end of the token list.
! 2034: **
! 2035: ** Additionally, doclist blobs are chunked into multiple segments,
! 2036: ** using segment to order the segments. New elements are added to
! 2037: ** the segment at segment 0, until it exceeds CHUNK_MAX. Then
! 2038: ** segment 0 is deleted, and the doclist is inserted at segment 1.
! 2039: ** If there is already a doclist at segment 1, the segment 0 doclist
! 2040: ** is merged with it, the segment 1 doclist is deleted, and the
! 2041: ** merged doclist is inserted at segment 2, repeating those
! 2042: ** operations until an insert succeeds.
! 2043: **
! 2044: ** Since this structure doesn't allow us to update elements in place
! 2045: ** in case of deletion or update, these are simply written to
! 2046: ** segment 0 (with an empty token list in case of deletion), with
! 2047: ** docListAccumulate() taking care to retain lower-segment
! 2048: ** information in preference to higher-segment information.
! 2049: */
! 2050: /* TODO(shess) Provide a VACUUM type operation which both removes
! 2051: ** deleted elements which are no longer necessary, and duplicated
! 2052: ** elements. I suspect this will probably not be necessary in
! 2053: ** practice, though.
! 2054: */
! 2055: static int fulltextCreate(sqlite3 *db, void *pAux,
! 2056: int argc, const char * const *argv,
! 2057: sqlite3_vtab **ppVTab, char **pzErr){
! 2058: int rc;
! 2059: TableSpec spec;
! 2060: StringBuffer schema;
! 2061: TRACE(("FTS1 Create\n"));
! 2062:
! 2063: rc = parseSpec(&spec, argc, argv, pzErr);
! 2064: if( rc!=SQLITE_OK ) return rc;
! 2065:
! 2066: initStringBuffer(&schema);
! 2067: append(&schema, "CREATE TABLE %_content(");
! 2068: appendList(&schema, spec.nColumn, spec.azContentColumn);
! 2069: append(&schema, ")");
! 2070: rc = sql_exec(db, spec.zDb, spec.zName, schema.s);
! 2071: free(schema.s);
! 2072: if( rc!=SQLITE_OK ) goto out;
! 2073:
! 2074: rc = sql_exec(db, spec.zDb, spec.zName,
! 2075: "create table %_term(term text, segment integer, doclist blob, "
! 2076: "primary key(term, segment));");
! 2077: if( rc!=SQLITE_OK ) goto out;
! 2078:
! 2079: rc = constructVtab(db, &spec, ppVTab, pzErr);
! 2080:
! 2081: out:
! 2082: clearTableSpec(&spec);
! 2083: return rc;
! 2084: }
! 2085:
! 2086: /* Decide how to handle an SQL query. */
! 2087: static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
! 2088: int i;
! 2089: TRACE(("FTS1 BestIndex\n"));
! 2090:
! 2091: for(i=0; i<pInfo->nConstraint; ++i){
! 2092: const struct sqlite3_index_constraint *pConstraint;
! 2093: pConstraint = &pInfo->aConstraint[i];
! 2094: if( pConstraint->usable ) {
! 2095: if( pConstraint->iColumn==-1 &&
! 2096: pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
! 2097: pInfo->idxNum = QUERY_ROWID; /* lookup by rowid */
! 2098: TRACE(("FTS1 QUERY_ROWID\n"));
! 2099: } else if( pConstraint->iColumn>=0 &&
! 2100: pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
! 2101: /* full-text search */
! 2102: pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
! 2103: TRACE(("FTS1 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
! 2104: } else continue;
! 2105:
! 2106: pInfo->aConstraintUsage[i].argvIndex = 1;
! 2107: pInfo->aConstraintUsage[i].omit = 1;
! 2108:
! 2109: /* An arbitrary value for now.
! 2110: * TODO: Perhaps rowid matches should be considered cheaper than
! 2111: * full-text searches. */
! 2112: pInfo->estimatedCost = 1.0;
! 2113:
! 2114: return SQLITE_OK;
! 2115: }
! 2116: }
! 2117: pInfo->idxNum = QUERY_GENERIC;
! 2118: return SQLITE_OK;
! 2119: }
! 2120:
! 2121: static int fulltextDisconnect(sqlite3_vtab *pVTab){
! 2122: TRACE(("FTS1 Disconnect %p\n", pVTab));
! 2123: fulltext_vtab_destroy((fulltext_vtab *)pVTab);
! 2124: return SQLITE_OK;
! 2125: }
! 2126:
! 2127: static int fulltextDestroy(sqlite3_vtab *pVTab){
! 2128: fulltext_vtab *v = (fulltext_vtab *)pVTab;
! 2129: int rc;
! 2130:
! 2131: TRACE(("FTS1 Destroy %p\n", pVTab));
! 2132: rc = sql_exec(v->db, v->zDb, v->zName,
! 2133: "drop table if exists %_content;"
! 2134: "drop table if exists %_term;"
! 2135: );
! 2136: if( rc!=SQLITE_OK ) return rc;
! 2137:
! 2138: fulltext_vtab_destroy((fulltext_vtab *)pVTab);
! 2139: return SQLITE_OK;
! 2140: }
! 2141:
! 2142: static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
! 2143: fulltext_cursor *c;
! 2144:
! 2145: c = (fulltext_cursor *) calloc(sizeof(fulltext_cursor), 1);
! 2146: /* sqlite will initialize c->base */
! 2147: *ppCursor = &c->base;
! 2148: TRACE(("FTS1 Open %p: %p\n", pVTab, c));
! 2149:
! 2150: return SQLITE_OK;
! 2151: }
! 2152:
! 2153:
! 2154: /* Free all of the dynamically allocated memory held by *q
! 2155: */
! 2156: static void queryClear(Query *q){
! 2157: int i;
! 2158: for(i = 0; i < q->nTerms; ++i){
! 2159: free(q->pTerms[i].pTerm);
! 2160: }
! 2161: free(q->pTerms);
! 2162: memset(q, 0, sizeof(*q));
! 2163: }
! 2164:
! 2165: /* Free all of the dynamically allocated memory held by the
! 2166: ** Snippet
! 2167: */
! 2168: static void snippetClear(Snippet *p){
! 2169: free(p->aMatch);
! 2170: free(p->zOffset);
! 2171: free(p->zSnippet);
! 2172: memset(p, 0, sizeof(*p));
! 2173: }
! 2174: /*
! 2175: ** Append a single entry to the p->aMatch[] log.
! 2176: */
! 2177: static void snippetAppendMatch(
! 2178: Snippet *p, /* Append the entry to this snippet */
! 2179: int iCol, int iTerm, /* The column and query term */
! 2180: int iStart, int nByte /* Offset and size of the match */
! 2181: ){
! 2182: int i;
! 2183: struct snippetMatch *pMatch;
! 2184: if( p->nMatch+1>=p->nAlloc ){
! 2185: p->nAlloc = p->nAlloc*2 + 10;
! 2186: p->aMatch = realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
! 2187: if( p->aMatch==0 ){
! 2188: p->nMatch = 0;
! 2189: p->nAlloc = 0;
! 2190: return;
! 2191: }
! 2192: }
! 2193: i = p->nMatch++;
! 2194: pMatch = &p->aMatch[i];
! 2195: pMatch->iCol = iCol;
! 2196: pMatch->iTerm = iTerm;
! 2197: pMatch->iStart = iStart;
! 2198: pMatch->nByte = nByte;
! 2199: }
! 2200:
! 2201: /*
! 2202: ** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
! 2203: */
! 2204: #define FTS1_ROTOR_SZ (32)
! 2205: #define FTS1_ROTOR_MASK (FTS1_ROTOR_SZ-1)
! 2206:
! 2207: /*
! 2208: ** Add entries to pSnippet->aMatch[] for every match that occurs against
! 2209: ** document zDoc[0..nDoc-1] which is stored in column iColumn.
! 2210: */
! 2211: static void snippetOffsetsOfColumn(
! 2212: Query *pQuery,
! 2213: Snippet *pSnippet,
! 2214: int iColumn,
! 2215: const char *zDoc,
! 2216: int nDoc
! 2217: ){
! 2218: const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */
! 2219: sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */
! 2220: sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */
! 2221: fulltext_vtab *pVtab; /* The full text index */
! 2222: int nColumn; /* Number of columns in the index */
! 2223: const QueryTerm *aTerm; /* Query string terms */
! 2224: int nTerm; /* Number of query string terms */
! 2225: int i, j; /* Loop counters */
! 2226: int rc; /* Return code */
! 2227: unsigned int match, prevMatch; /* Phrase search bitmasks */
! 2228: const char *zToken; /* Next token from the tokenizer */
! 2229: int nToken; /* Size of zToken */
! 2230: int iBegin, iEnd, iPos; /* Offsets of beginning and end */
! 2231:
! 2232: /* The following variables keep a circular buffer of the last
! 2233: ** few tokens */
! 2234: unsigned int iRotor = 0; /* Index of current token */
! 2235: int iRotorBegin[FTS1_ROTOR_SZ]; /* Beginning offset of token */
! 2236: int iRotorLen[FTS1_ROTOR_SZ]; /* Length of token */
! 2237:
! 2238: pVtab = pQuery->pFts;
! 2239: nColumn = pVtab->nColumn;
! 2240: pTokenizer = pVtab->pTokenizer;
! 2241: pTModule = pTokenizer->pModule;
! 2242: rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
! 2243: if( rc ) return;
! 2244: pTCursor->pTokenizer = pTokenizer;
! 2245: aTerm = pQuery->pTerms;
! 2246: nTerm = pQuery->nTerms;
! 2247: if( nTerm>=FTS1_ROTOR_SZ ){
! 2248: nTerm = FTS1_ROTOR_SZ - 1;
! 2249: }
! 2250: prevMatch = 0;
! 2251: while(1){
! 2252: rc = pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
! 2253: if( rc ) break;
! 2254: iRotorBegin[iRotor&FTS1_ROTOR_MASK] = iBegin;
! 2255: iRotorLen[iRotor&FTS1_ROTOR_MASK] = iEnd-iBegin;
! 2256: match = 0;
! 2257: for(i=0; i<nTerm; i++){
! 2258: int iCol;
! 2259: iCol = aTerm[i].iColumn;
! 2260: if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
! 2261: if( aTerm[i].nTerm!=nToken ) continue;
! 2262: if( memcmp(aTerm[i].pTerm, zToken, nToken) ) continue;
! 2263: if( aTerm[i].iPhrase>1 && (prevMatch & (1<<i))==0 ) continue;
! 2264: match |= 1<<i;
! 2265: if( i==nTerm-1 || aTerm[i+1].iPhrase==1 ){
! 2266: for(j=aTerm[i].iPhrase-1; j>=0; j--){
! 2267: int k = (iRotor-j) & FTS1_ROTOR_MASK;
! 2268: snippetAppendMatch(pSnippet, iColumn, i-j,
! 2269: iRotorBegin[k], iRotorLen[k]);
! 2270: }
! 2271: }
! 2272: }
! 2273: prevMatch = match<<1;
! 2274: iRotor++;
! 2275: }
! 2276: pTModule->xClose(pTCursor);
! 2277: }
! 2278:
! 2279:
! 2280: /*
! 2281: ** Compute all offsets for the current row of the query.
! 2282: ** If the offsets have already been computed, this routine is a no-op.
! 2283: */
! 2284: static void snippetAllOffsets(fulltext_cursor *p){
! 2285: int nColumn;
! 2286: int iColumn, i;
! 2287: int iFirst, iLast;
! 2288: fulltext_vtab *pFts;
! 2289:
! 2290: if( p->snippet.nMatch ) return;
! 2291: if( p->q.nTerms==0 ) return;
! 2292: pFts = p->q.pFts;
! 2293: nColumn = pFts->nColumn;
! 2294: iColumn = p->iCursorType - QUERY_FULLTEXT;
! 2295: if( iColumn<0 || iColumn>=nColumn ){
! 2296: iFirst = 0;
! 2297: iLast = nColumn-1;
! 2298: }else{
! 2299: iFirst = iColumn;
! 2300: iLast = iColumn;
! 2301: }
! 2302: for(i=iFirst; i<=iLast; i++){
! 2303: const char *zDoc;
! 2304: int nDoc;
! 2305: zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
! 2306: nDoc = sqlite3_column_bytes(p->pStmt, i+1);
! 2307: snippetOffsetsOfColumn(&p->q, &p->snippet, i, zDoc, nDoc);
! 2308: }
! 2309: }
! 2310:
! 2311: /*
! 2312: ** Convert the information in the aMatch[] array of the snippet
! 2313: ** into the string zOffset[0..nOffset-1].
! 2314: */
! 2315: static void snippetOffsetText(Snippet *p){
! 2316: int i;
! 2317: int cnt = 0;
! 2318: StringBuffer sb;
! 2319: char zBuf[200];
! 2320: if( p->zOffset ) return;
! 2321: initStringBuffer(&sb);
! 2322: for(i=0; i<p->nMatch; i++){
! 2323: struct snippetMatch *pMatch = &p->aMatch[i];
! 2324: zBuf[0] = ' ';
! 2325: sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
! 2326: pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
! 2327: append(&sb, zBuf);
! 2328: cnt++;
! 2329: }
! 2330: p->zOffset = sb.s;
! 2331: p->nOffset = sb.len;
! 2332: }
! 2333:
! 2334: /*
! 2335: ** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
! 2336: ** of matching words some of which might be in zDoc. zDoc is column
! 2337: ** number iCol.
! 2338: **
! 2339: ** iBreak is suggested spot in zDoc where we could begin or end an
! 2340: ** excerpt. Return a value similar to iBreak but possibly adjusted
! 2341: ** to be a little left or right so that the break point is better.
! 2342: */
! 2343: static int wordBoundary(
! 2344: int iBreak, /* The suggested break point */
! 2345: const char *zDoc, /* Document text */
! 2346: int nDoc, /* Number of bytes in zDoc[] */
! 2347: struct snippetMatch *aMatch, /* Matching words */
! 2348: int nMatch, /* Number of entries in aMatch[] */
! 2349: int iCol /* The column number for zDoc[] */
! 2350: ){
! 2351: int i;
! 2352: if( iBreak<=10 ){
! 2353: return 0;
! 2354: }
! 2355: if( iBreak>=nDoc-10 ){
! 2356: return nDoc;
! 2357: }
! 2358: for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
! 2359: while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
! 2360: if( i<nMatch ){
! 2361: if( aMatch[i].iStart<iBreak+10 ){
! 2362: return aMatch[i].iStart;
! 2363: }
! 2364: if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
! 2365: return aMatch[i-1].iStart;
! 2366: }
! 2367: }
! 2368: for(i=1; i<=10; i++){
! 2369: if( safe_isspace(zDoc[iBreak-i]) ){
! 2370: return iBreak - i + 1;
! 2371: }
! 2372: if( safe_isspace(zDoc[iBreak+i]) ){
! 2373: return iBreak + i + 1;
! 2374: }
! 2375: }
! 2376: return iBreak;
! 2377: }
! 2378:
! 2379: /*
! 2380: ** If the StringBuffer does not end in white space, add a single
! 2381: ** space character to the end.
! 2382: */
! 2383: static void appendWhiteSpace(StringBuffer *p){
! 2384: if( p->len==0 ) return;
! 2385: if( safe_isspace(p->s[p->len-1]) ) return;
! 2386: append(p, " ");
! 2387: }
! 2388:
! 2389: /*
! 2390: ** Remove white space from teh end of the StringBuffer
! 2391: */
! 2392: static void trimWhiteSpace(StringBuffer *p){
! 2393: while( p->len>0 && safe_isspace(p->s[p->len-1]) ){
! 2394: p->len--;
! 2395: }
! 2396: }
! 2397:
! 2398:
! 2399:
! 2400: /*
! 2401: ** Allowed values for Snippet.aMatch[].snStatus
! 2402: */
! 2403: #define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
! 2404: #define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
! 2405:
! 2406: /*
! 2407: ** Generate the text of a snippet.
! 2408: */
! 2409: static void snippetText(
! 2410: fulltext_cursor *pCursor, /* The cursor we need the snippet for */
! 2411: const char *zStartMark, /* Markup to appear before each match */
! 2412: const char *zEndMark, /* Markup to appear after each match */
! 2413: const char *zEllipsis /* Ellipsis mark */
! 2414: ){
! 2415: int i, j;
! 2416: struct snippetMatch *aMatch;
! 2417: int nMatch;
! 2418: int nDesired;
! 2419: StringBuffer sb;
! 2420: int tailCol;
! 2421: int tailOffset;
! 2422: int iCol;
! 2423: int nDoc;
! 2424: const char *zDoc;
! 2425: int iStart, iEnd;
! 2426: int tailEllipsis = 0;
! 2427: int iMatch;
! 2428:
! 2429:
! 2430: free(pCursor->snippet.zSnippet);
! 2431: pCursor->snippet.zSnippet = 0;
! 2432: aMatch = pCursor->snippet.aMatch;
! 2433: nMatch = pCursor->snippet.nMatch;
! 2434: initStringBuffer(&sb);
! 2435:
! 2436: for(i=0; i<nMatch; i++){
! 2437: aMatch[i].snStatus = SNIPPET_IGNORE;
! 2438: }
! 2439: nDesired = 0;
! 2440: for(i=0; i<pCursor->q.nTerms; i++){
! 2441: for(j=0; j<nMatch; j++){
! 2442: if( aMatch[j].iTerm==i ){
! 2443: aMatch[j].snStatus = SNIPPET_DESIRED;
! 2444: nDesired++;
! 2445: break;
! 2446: }
! 2447: }
! 2448: }
! 2449:
! 2450: iMatch = 0;
! 2451: tailCol = -1;
! 2452: tailOffset = 0;
! 2453: for(i=0; i<nMatch && nDesired>0; i++){
! 2454: if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
! 2455: nDesired--;
! 2456: iCol = aMatch[i].iCol;
! 2457: zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
! 2458: nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
! 2459: iStart = aMatch[i].iStart - 40;
! 2460: iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
! 2461: if( iStart<=10 ){
! 2462: iStart = 0;
! 2463: }
! 2464: if( iCol==tailCol && iStart<=tailOffset+20 ){
! 2465: iStart = tailOffset;
! 2466: }
! 2467: if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
! 2468: trimWhiteSpace(&sb);
! 2469: appendWhiteSpace(&sb);
! 2470: append(&sb, zEllipsis);
! 2471: appendWhiteSpace(&sb);
! 2472: }
! 2473: iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
! 2474: iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
! 2475: if( iEnd>=nDoc-10 ){
! 2476: iEnd = nDoc;
! 2477: tailEllipsis = 0;
! 2478: }else{
! 2479: tailEllipsis = 1;
! 2480: }
! 2481: while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
! 2482: while( iStart<iEnd ){
! 2483: while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
! 2484: && aMatch[iMatch].iCol<=iCol ){
! 2485: iMatch++;
! 2486: }
! 2487: if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
! 2488: && aMatch[iMatch].iCol==iCol ){
! 2489: nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
! 2490: iStart = aMatch[iMatch].iStart;
! 2491: append(&sb, zStartMark);
! 2492: nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
! 2493: append(&sb, zEndMark);
! 2494: iStart += aMatch[iMatch].nByte;
! 2495: for(j=iMatch+1; j<nMatch; j++){
! 2496: if( aMatch[j].iTerm==aMatch[iMatch].iTerm
! 2497: && aMatch[j].snStatus==SNIPPET_DESIRED ){
! 2498: nDesired--;
! 2499: aMatch[j].snStatus = SNIPPET_IGNORE;
! 2500: }
! 2501: }
! 2502: }else{
! 2503: nappend(&sb, &zDoc[iStart], iEnd - iStart);
! 2504: iStart = iEnd;
! 2505: }
! 2506: }
! 2507: tailCol = iCol;
! 2508: tailOffset = iEnd;
! 2509: }
! 2510: trimWhiteSpace(&sb);
! 2511: if( tailEllipsis ){
! 2512: appendWhiteSpace(&sb);
! 2513: append(&sb, zEllipsis);
! 2514: }
! 2515: pCursor->snippet.zSnippet = sb.s;
! 2516: pCursor->snippet.nSnippet = sb.len;
! 2517: }
! 2518:
! 2519:
! 2520: /*
! 2521: ** Close the cursor. For additional information see the documentation
! 2522: ** on the xClose method of the virtual table interface.
! 2523: */
! 2524: static int fulltextClose(sqlite3_vtab_cursor *pCursor){
! 2525: fulltext_cursor *c = (fulltext_cursor *) pCursor;
! 2526: TRACE(("FTS1 Close %p\n", c));
! 2527: sqlite3_finalize(c->pStmt);
! 2528: queryClear(&c->q);
! 2529: snippetClear(&c->snippet);
! 2530: if( c->result.pDoclist!=NULL ){
! 2531: docListDelete(c->result.pDoclist);
! 2532: }
! 2533: free(c);
! 2534: return SQLITE_OK;
! 2535: }
! 2536:
! 2537: static int fulltextNext(sqlite3_vtab_cursor *pCursor){
! 2538: fulltext_cursor *c = (fulltext_cursor *) pCursor;
! 2539: sqlite_int64 iDocid;
! 2540: int rc;
! 2541:
! 2542: TRACE(("FTS1 Next %p\n", pCursor));
! 2543: snippetClear(&c->snippet);
! 2544: if( c->iCursorType < QUERY_FULLTEXT ){
! 2545: /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
! 2546: rc = sqlite3_step(c->pStmt);
! 2547: switch( rc ){
! 2548: case SQLITE_ROW:
! 2549: c->eof = 0;
! 2550: return SQLITE_OK;
! 2551: case SQLITE_DONE:
! 2552: c->eof = 1;
! 2553: return SQLITE_OK;
! 2554: default:
! 2555: c->eof = 1;
! 2556: return rc;
! 2557: }
! 2558: } else { /* full-text query */
! 2559: rc = sqlite3_reset(c->pStmt);
! 2560: if( rc!=SQLITE_OK ) return rc;
! 2561:
! 2562: iDocid = nextDocid(&c->result);
! 2563: if( iDocid==0 ){
! 2564: c->eof = 1;
! 2565: return SQLITE_OK;
! 2566: }
! 2567: rc = sqlite3_bind_int64(c->pStmt, 1, iDocid);
! 2568: if( rc!=SQLITE_OK ) return rc;
! 2569: /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
! 2570: rc = sqlite3_step(c->pStmt);
! 2571: if( rc==SQLITE_ROW ){ /* the case we expect */
! 2572: c->eof = 0;
! 2573: return SQLITE_OK;
! 2574: }
! 2575: /* an error occurred; abort */
! 2576: return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
! 2577: }
! 2578: }
! 2579:
! 2580:
! 2581: /* Return a DocList corresponding to the query term *pTerm. If *pTerm
! 2582: ** is the first term of a phrase query, go ahead and evaluate the phrase
! 2583: ** query and return the doclist for the entire phrase query.
! 2584: **
! 2585: ** The result is stored in pTerm->doclist.
! 2586: */
! 2587: static int docListOfTerm(
! 2588: fulltext_vtab *v, /* The full text index */
! 2589: int iColumn, /* column to restrict to. No restrition if >=nColumn */
! 2590: QueryTerm *pQTerm, /* Term we are looking for, or 1st term of a phrase */
! 2591: DocList **ppResult /* Write the result here */
! 2592: ){
! 2593: DocList *pLeft, *pRight, *pNew;
! 2594: int i, rc;
! 2595:
! 2596: pLeft = docListNew(DL_POSITIONS);
! 2597: rc = term_select_all(v, iColumn, pQTerm->pTerm, pQTerm->nTerm, pLeft);
! 2598: if( rc ){
! 2599: docListDelete(pLeft);
! 2600: return rc;
! 2601: }
! 2602: for(i=1; i<=pQTerm->nPhrase; i++){
! 2603: pRight = docListNew(DL_POSITIONS);
! 2604: rc = term_select_all(v, iColumn, pQTerm[i].pTerm, pQTerm[i].nTerm, pRight);
! 2605: if( rc ){
! 2606: docListDelete(pLeft);
! 2607: return rc;
! 2608: }
! 2609: pNew = docListNew(i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS);
! 2610: docListPhraseMerge(pLeft, pRight, pNew);
! 2611: docListDelete(pLeft);
! 2612: docListDelete(pRight);
! 2613: pLeft = pNew;
! 2614: }
! 2615: *ppResult = pLeft;
! 2616: return SQLITE_OK;
! 2617: }
! 2618:
! 2619: /* Add a new term pTerm[0..nTerm-1] to the query *q.
! 2620: */
! 2621: static void queryAdd(Query *q, const char *pTerm, int nTerm){
! 2622: QueryTerm *t;
! 2623: ++q->nTerms;
! 2624: q->pTerms = realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0]));
! 2625: if( q->pTerms==0 ){
! 2626: q->nTerms = 0;
! 2627: return;
! 2628: }
! 2629: t = &q->pTerms[q->nTerms - 1];
! 2630: memset(t, 0, sizeof(*t));
! 2631: t->pTerm = malloc(nTerm+1);
! 2632: memcpy(t->pTerm, pTerm, nTerm);
! 2633: t->pTerm[nTerm] = 0;
! 2634: t->nTerm = nTerm;
! 2635: t->isOr = q->nextIsOr;
! 2636: q->nextIsOr = 0;
! 2637: t->iColumn = q->nextColumn;
! 2638: q->nextColumn = q->dfltColumn;
! 2639: }
! 2640:
! 2641: /*
! 2642: ** Check to see if the string zToken[0...nToken-1] matches any
! 2643: ** column name in the virtual table. If it does,
! 2644: ** return the zero-indexed column number. If not, return -1.
! 2645: */
! 2646: static int checkColumnSpecifier(
! 2647: fulltext_vtab *pVtab, /* The virtual table */
! 2648: const char *zToken, /* Text of the token */
! 2649: int nToken /* Number of characters in the token */
! 2650: ){
! 2651: int i;
! 2652: for(i=0; i<pVtab->nColumn; i++){
! 2653: if( memcmp(pVtab->azColumn[i], zToken, nToken)==0
! 2654: && pVtab->azColumn[i][nToken]==0 ){
! 2655: return i;
! 2656: }
! 2657: }
! 2658: return -1;
! 2659: }
! 2660:
! 2661: /*
! 2662: ** Parse the text at pSegment[0..nSegment-1]. Add additional terms
! 2663: ** to the query being assemblied in pQuery.
! 2664: **
! 2665: ** inPhrase is true if pSegment[0..nSegement-1] is contained within
! 2666: ** double-quotes. If inPhrase is true, then the first term
! 2667: ** is marked with the number of terms in the phrase less one and
! 2668: ** OR and "-" syntax is ignored. If inPhrase is false, then every
! 2669: ** term found is marked with nPhrase=0 and OR and "-" syntax is significant.
! 2670: */
! 2671: static int tokenizeSegment(
! 2672: sqlite3_tokenizer *pTokenizer, /* The tokenizer to use */
! 2673: const char *pSegment, int nSegment, /* Query expression being parsed */
! 2674: int inPhrase, /* True if within "..." */
! 2675: Query *pQuery /* Append results here */
! 2676: ){
! 2677: const sqlite3_tokenizer_module *pModule = pTokenizer->pModule;
! 2678: sqlite3_tokenizer_cursor *pCursor;
! 2679: int firstIndex = pQuery->nTerms;
! 2680: int iCol;
! 2681: int nTerm = 1;
! 2682:
! 2683: int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
! 2684: if( rc!=SQLITE_OK ) return rc;
! 2685: pCursor->pTokenizer = pTokenizer;
! 2686:
! 2687: while( 1 ){
! 2688: const char *pToken;
! 2689: int nToken, iBegin, iEnd, iPos;
! 2690:
! 2691: rc = pModule->xNext(pCursor,
! 2692: &pToken, &nToken,
! 2693: &iBegin, &iEnd, &iPos);
! 2694: if( rc!=SQLITE_OK ) break;
! 2695: if( !inPhrase &&
! 2696: pSegment[iEnd]==':' &&
! 2697: (iCol = checkColumnSpecifier(pQuery->pFts, pToken, nToken))>=0 ){
! 2698: pQuery->nextColumn = iCol;
! 2699: continue;
! 2700: }
! 2701: if( !inPhrase && pQuery->nTerms>0 && nToken==2
! 2702: && pSegment[iBegin]=='O' && pSegment[iBegin+1]=='R' ){
! 2703: pQuery->nextIsOr = 1;
! 2704: continue;
! 2705: }
! 2706: queryAdd(pQuery, pToken, nToken);
! 2707: if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
! 2708: pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
! 2709: }
! 2710: pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm;
! 2711: if( inPhrase ){
! 2712: nTerm++;
! 2713: }
! 2714: }
! 2715:
! 2716: if( inPhrase && pQuery->nTerms>firstIndex ){
! 2717: pQuery->pTerms[firstIndex].nPhrase = pQuery->nTerms - firstIndex - 1;
! 2718: }
! 2719:
! 2720: return pModule->xClose(pCursor);
! 2721: }
! 2722:
! 2723: /* Parse a query string, yielding a Query object pQuery.
! 2724: **
! 2725: ** The calling function will need to queryClear() to clean up
! 2726: ** the dynamically allocated memory held by pQuery.
! 2727: */
! 2728: static int parseQuery(
! 2729: fulltext_vtab *v, /* The fulltext index */
! 2730: const char *zInput, /* Input text of the query string */
! 2731: int nInput, /* Size of the input text */
! 2732: int dfltColumn, /* Default column of the index to match against */
! 2733: Query *pQuery /* Write the parse results here. */
! 2734: ){
! 2735: int iInput, inPhrase = 0;
! 2736:
! 2737: if( zInput==0 ) nInput = 0;
! 2738: if( nInput<0 ) nInput = strlen(zInput);
! 2739: pQuery->nTerms = 0;
! 2740: pQuery->pTerms = NULL;
! 2741: pQuery->nextIsOr = 0;
! 2742: pQuery->nextColumn = dfltColumn;
! 2743: pQuery->dfltColumn = dfltColumn;
! 2744: pQuery->pFts = v;
! 2745:
! 2746: for(iInput=0; iInput<nInput; ++iInput){
! 2747: int i;
! 2748: for(i=iInput; i<nInput && zInput[i]!='"'; ++i){}
! 2749: if( i>iInput ){
! 2750: tokenizeSegment(v->pTokenizer, zInput+iInput, i-iInput, inPhrase,
! 2751: pQuery);
! 2752: }
! 2753: iInput = i;
! 2754: if( i<nInput ){
! 2755: assert( zInput[i]=='"' );
! 2756: inPhrase = !inPhrase;
! 2757: }
! 2758: }
! 2759:
! 2760: if( inPhrase ){
! 2761: /* unmatched quote */
! 2762: queryClear(pQuery);
! 2763: return SQLITE_ERROR;
! 2764: }
! 2765: return SQLITE_OK;
! 2766: }
! 2767:
! 2768: /* Perform a full-text query using the search expression in
! 2769: ** zInput[0..nInput-1]. Return a list of matching documents
! 2770: ** in pResult.
! 2771: **
! 2772: ** Queries must match column iColumn. Or if iColumn>=nColumn
! 2773: ** they are allowed to match against any column.
! 2774: */
! 2775: static int fulltextQuery(
! 2776: fulltext_vtab *v, /* The full text index */
! 2777: int iColumn, /* Match against this column by default */
! 2778: const char *zInput, /* The query string */
! 2779: int nInput, /* Number of bytes in zInput[] */
! 2780: DocList **pResult, /* Write the result doclist here */
! 2781: Query *pQuery /* Put parsed query string here */
! 2782: ){
! 2783: int i, iNext, rc;
! 2784: DocList *pLeft = NULL;
! 2785: DocList *pRight, *pNew, *pOr;
! 2786: int nNot = 0;
! 2787: QueryTerm *aTerm;
! 2788:
! 2789: rc = parseQuery(v, zInput, nInput, iColumn, pQuery);
! 2790: if( rc!=SQLITE_OK ) return rc;
! 2791:
! 2792: /* Merge AND terms. */
! 2793: aTerm = pQuery->pTerms;
! 2794: for(i = 0; i<pQuery->nTerms; i=iNext){
! 2795: if( aTerm[i].isNot ){
! 2796: /* Handle all NOT terms in a separate pass */
! 2797: nNot++;
! 2798: iNext = i + aTerm[i].nPhrase+1;
! 2799: continue;
! 2800: }
! 2801: iNext = i + aTerm[i].nPhrase + 1;
! 2802: rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &pRight);
! 2803: if( rc ){
! 2804: queryClear(pQuery);
! 2805: return rc;
! 2806: }
! 2807: while( iNext<pQuery->nTerms && aTerm[iNext].isOr ){
! 2808: rc = docListOfTerm(v, aTerm[iNext].iColumn, &aTerm[iNext], &pOr);
! 2809: iNext += aTerm[iNext].nPhrase + 1;
! 2810: if( rc ){
! 2811: queryClear(pQuery);
! 2812: return rc;
! 2813: }
! 2814: pNew = docListNew(DL_DOCIDS);
! 2815: docListOrMerge(pRight, pOr, pNew);
! 2816: docListDelete(pRight);
! 2817: docListDelete(pOr);
! 2818: pRight = pNew;
! 2819: }
! 2820: if( pLeft==0 ){
! 2821: pLeft = pRight;
! 2822: }else{
! 2823: pNew = docListNew(DL_DOCIDS);
! 2824: docListAndMerge(pLeft, pRight, pNew);
! 2825: docListDelete(pRight);
! 2826: docListDelete(pLeft);
! 2827: pLeft = pNew;
! 2828: }
! 2829: }
! 2830:
! 2831: if( nNot && pLeft==0 ){
! 2832: /* We do not yet know how to handle a query of only NOT terms */
! 2833: return SQLITE_ERROR;
! 2834: }
! 2835:
! 2836: /* Do the EXCEPT terms */
! 2837: for(i=0; i<pQuery->nTerms; i += aTerm[i].nPhrase + 1){
! 2838: if( !aTerm[i].isNot ) continue;
! 2839: rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &pRight);
! 2840: if( rc ){
! 2841: queryClear(pQuery);
! 2842: docListDelete(pLeft);
! 2843: return rc;
! 2844: }
! 2845: pNew = docListNew(DL_DOCIDS);
! 2846: docListExceptMerge(pLeft, pRight, pNew);
! 2847: docListDelete(pRight);
! 2848: docListDelete(pLeft);
! 2849: pLeft = pNew;
! 2850: }
! 2851:
! 2852: *pResult = pLeft;
! 2853: return rc;
! 2854: }
! 2855:
! 2856: /*
! 2857: ** This is the xFilter interface for the virtual table. See
! 2858: ** the virtual table xFilter method documentation for additional
! 2859: ** information.
! 2860: **
! 2861: ** If idxNum==QUERY_GENERIC then do a full table scan against
! 2862: ** the %_content table.
! 2863: **
! 2864: ** If idxNum==QUERY_ROWID then do a rowid lookup for a single entry
! 2865: ** in the %_content table.
! 2866: **
! 2867: ** If idxNum>=QUERY_FULLTEXT then use the full text index. The
! 2868: ** column on the left-hand side of the MATCH operator is column
! 2869: ** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
! 2870: ** side of the MATCH operator.
! 2871: */
! 2872: /* TODO(shess) Upgrade the cursor initialization and destruction to
! 2873: ** account for fulltextFilter() being called multiple times on the
! 2874: ** same cursor. The current solution is very fragile. Apply fix to
! 2875: ** fts2 as appropriate.
! 2876: */
! 2877: static int fulltextFilter(
! 2878: sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
! 2879: int idxNum, const char *idxStr, /* Which indexing scheme to use */
! 2880: int argc, sqlite3_value **argv /* Arguments for the indexing scheme */
! 2881: ){
! 2882: fulltext_cursor *c = (fulltext_cursor *) pCursor;
! 2883: fulltext_vtab *v = cursor_vtab(c);
! 2884: int rc;
! 2885: char *zSql;
! 2886:
! 2887: TRACE(("FTS1 Filter %p\n",pCursor));
! 2888:
! 2889: zSql = sqlite3_mprintf("select rowid, * from %%_content %s",
! 2890: idxNum==QUERY_GENERIC ? "" : "where rowid=?");
! 2891: sqlite3_finalize(c->pStmt);
! 2892: rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, zSql);
! 2893: sqlite3_free(zSql);
! 2894: if( rc!=SQLITE_OK ) return rc;
! 2895:
! 2896: c->iCursorType = idxNum;
! 2897: switch( idxNum ){
! 2898: case QUERY_GENERIC:
! 2899: break;
! 2900:
! 2901: case QUERY_ROWID:
! 2902: rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
! 2903: if( rc!=SQLITE_OK ) return rc;
! 2904: break;
! 2905:
! 2906: default: /* full-text search */
! 2907: {
! 2908: const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
! 2909: DocList *pResult;
! 2910: assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
! 2911: assert( argc==1 );
! 2912: queryClear(&c->q);
! 2913: rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &pResult, &c->q);
! 2914: if( rc!=SQLITE_OK ) return rc;
! 2915: if( c->result.pDoclist!=NULL ) docListDelete(c->result.pDoclist);
! 2916: readerInit(&c->result, pResult);
! 2917: break;
! 2918: }
! 2919: }
! 2920:
! 2921: return fulltextNext(pCursor);
! 2922: }
! 2923:
! 2924: /* This is the xEof method of the virtual table. The SQLite core
! 2925: ** calls this routine to find out if it has reached the end of
! 2926: ** a query's results set.
! 2927: */
! 2928: static int fulltextEof(sqlite3_vtab_cursor *pCursor){
! 2929: fulltext_cursor *c = (fulltext_cursor *) pCursor;
! 2930: return c->eof;
! 2931: }
! 2932:
! 2933: /* This is the xColumn method of the virtual table. The SQLite
! 2934: ** core calls this method during a query when it needs the value
! 2935: ** of a column from the virtual table. This method needs to use
! 2936: ** one of the sqlite3_result_*() routines to store the requested
! 2937: ** value back in the pContext.
! 2938: */
! 2939: static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
! 2940: sqlite3_context *pContext, int idxCol){
! 2941: fulltext_cursor *c = (fulltext_cursor *) pCursor;
! 2942: fulltext_vtab *v = cursor_vtab(c);
! 2943:
! 2944: if( idxCol<v->nColumn ){
! 2945: sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
! 2946: sqlite3_result_value(pContext, pVal);
! 2947: }else if( idxCol==v->nColumn ){
! 2948: /* The extra column whose name is the same as the table.
! 2949: ** Return a blob which is a pointer to the cursor
! 2950: */
! 2951: sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
! 2952: }
! 2953: return SQLITE_OK;
! 2954: }
! 2955:
! 2956: /* This is the xRowid method. The SQLite core calls this routine to
! 2957: ** retrive the rowid for the current row of the result set. The
! 2958: ** rowid should be written to *pRowid.
! 2959: */
! 2960: static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
! 2961: fulltext_cursor *c = (fulltext_cursor *) pCursor;
! 2962:
! 2963: *pRowid = sqlite3_column_int64(c->pStmt, 0);
! 2964: return SQLITE_OK;
! 2965: }
! 2966:
! 2967: /* Add all terms in [zText] to the given hash table. If [iColumn] > 0,
! 2968: * we also store positions and offsets in the hash table using the given
! 2969: * column number. */
! 2970: static int buildTerms(fulltext_vtab *v, fts1Hash *terms, sqlite_int64 iDocid,
! 2971: const char *zText, int iColumn){
! 2972: sqlite3_tokenizer *pTokenizer = v->pTokenizer;
! 2973: sqlite3_tokenizer_cursor *pCursor;
! 2974: const char *pToken;
! 2975: int nTokenBytes;
! 2976: int iStartOffset, iEndOffset, iPosition;
! 2977: int rc;
! 2978:
! 2979: rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
! 2980: if( rc!=SQLITE_OK ) return rc;
! 2981:
! 2982: pCursor->pTokenizer = pTokenizer;
! 2983: while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
! 2984: &pToken, &nTokenBytes,
! 2985: &iStartOffset, &iEndOffset,
! 2986: &iPosition) ){
! 2987: DocList *p;
! 2988:
! 2989: /* Positions can't be negative; we use -1 as a terminator internally. */
! 2990: if( iPosition<0 ){
! 2991: pTokenizer->pModule->xClose(pCursor);
! 2992: return SQLITE_ERROR;
! 2993: }
! 2994:
! 2995: p = fts1HashFind(terms, pToken, nTokenBytes);
! 2996: if( p==NULL ){
! 2997: p = docListNew(DL_DEFAULT);
! 2998: docListAddDocid(p, iDocid);
! 2999: fts1HashInsert(terms, pToken, nTokenBytes, p);
! 3000: }
! 3001: if( iColumn>=0 ){
! 3002: docListAddPosOffset(p, iColumn, iPosition, iStartOffset, iEndOffset);
! 3003: }
! 3004: }
! 3005:
! 3006: /* TODO(shess) Check return? Should this be able to cause errors at
! 3007: ** this point? Actually, same question about sqlite3_finalize(),
! 3008: ** though one could argue that failure there means that the data is
! 3009: ** not durable. *ponder*
! 3010: */
! 3011: pTokenizer->pModule->xClose(pCursor);
! 3012: return rc;
! 3013: }
! 3014:
! 3015: /* Update the %_terms table to map the term [pTerm] to the given rowid. */
! 3016: static int index_insert_term(fulltext_vtab *v, const char *pTerm, int nTerm,
! 3017: DocList *d){
! 3018: sqlite_int64 iIndexRow;
! 3019: DocList doclist;
! 3020: int iSegment = 0, rc;
! 3021:
! 3022: rc = term_select(v, pTerm, nTerm, iSegment, &iIndexRow, &doclist);
! 3023: if( rc==SQLITE_DONE ){
! 3024: docListInit(&doclist, DL_DEFAULT, 0, 0);
! 3025: docListUpdate(&doclist, d);
! 3026: /* TODO(shess) Consider length(doclist)>CHUNK_MAX? */
! 3027: rc = term_insert(v, NULL, pTerm, nTerm, iSegment, &doclist);
! 3028: goto err;
! 3029: }
! 3030: if( rc!=SQLITE_ROW ) return SQLITE_ERROR;
! 3031:
! 3032: docListUpdate(&doclist, d);
! 3033: if( doclist.nData<=CHUNK_MAX ){
! 3034: rc = term_update(v, iIndexRow, &doclist);
! 3035: goto err;
! 3036: }
! 3037:
! 3038: /* Doclist doesn't fit, delete what's there, and accumulate
! 3039: ** forward.
! 3040: */
! 3041: rc = term_delete(v, iIndexRow);
! 3042: if( rc!=SQLITE_OK ) goto err;
! 3043:
! 3044: /* Try to insert the doclist into a higher segment bucket. On
! 3045: ** failure, accumulate existing doclist with the doclist from that
! 3046: ** bucket, and put results in the next bucket.
! 3047: */
! 3048: iSegment++;
! 3049: while( (rc=term_insert(v, &iIndexRow, pTerm, nTerm, iSegment,
! 3050: &doclist))!=SQLITE_OK ){
! 3051: sqlite_int64 iSegmentRow;
! 3052: DocList old;
! 3053: int rc2;
! 3054:
! 3055: /* Retain old error in case the term_insert() error was really an
! 3056: ** error rather than a bounced insert.
! 3057: */
! 3058: rc2 = term_select(v, pTerm, nTerm, iSegment, &iSegmentRow, &old);
! 3059: if( rc2!=SQLITE_ROW ) goto err;
! 3060:
! 3061: rc = term_delete(v, iSegmentRow);
! 3062: if( rc!=SQLITE_OK ) goto err;
! 3063:
! 3064: /* Reusing lowest-number deleted row keeps the index smaller. */
! 3065: if( iSegmentRow<iIndexRow ) iIndexRow = iSegmentRow;
! 3066:
! 3067: /* doclist contains the newer data, so accumulate it over old.
! 3068: ** Then steal accumulated data for doclist.
! 3069: */
! 3070: docListAccumulate(&old, &doclist);
! 3071: docListDestroy(&doclist);
! 3072: doclist = old;
! 3073:
! 3074: iSegment++;
! 3075: }
! 3076:
! 3077: err:
! 3078: docListDestroy(&doclist);
! 3079: return rc;
! 3080: }
! 3081:
! 3082: /* Add doclists for all terms in [pValues] to the hash table [terms]. */
! 3083: static int insertTerms(fulltext_vtab *v, fts1Hash *terms, sqlite_int64 iRowid,
! 3084: sqlite3_value **pValues){
! 3085: int i;
! 3086: for(i = 0; i < v->nColumn ; ++i){
! 3087: char *zText = (char*)sqlite3_value_text(pValues[i]);
! 3088: int rc = buildTerms(v, terms, iRowid, zText, i);
! 3089: if( rc!=SQLITE_OK ) return rc;
! 3090: }
! 3091: return SQLITE_OK;
! 3092: }
! 3093:
! 3094: /* Add empty doclists for all terms in the given row's content to the hash
! 3095: * table [pTerms]. */
! 3096: static int deleteTerms(fulltext_vtab *v, fts1Hash *pTerms, sqlite_int64 iRowid){
! 3097: const char **pValues;
! 3098: int i;
! 3099:
! 3100: int rc = content_select(v, iRowid, &pValues);
! 3101: if( rc!=SQLITE_OK ) return rc;
! 3102:
! 3103: for(i = 0 ; i < v->nColumn; ++i) {
! 3104: rc = buildTerms(v, pTerms, iRowid, pValues[i], -1);
! 3105: if( rc!=SQLITE_OK ) break;
! 3106: }
! 3107:
! 3108: freeStringArray(v->nColumn, pValues);
! 3109: return SQLITE_OK;
! 3110: }
! 3111:
! 3112: /* Insert a row into the %_content table; set *piRowid to be the ID of the
! 3113: * new row. Fill [pTerms] with new doclists for the %_term table. */
! 3114: static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestRowid,
! 3115: sqlite3_value **pValues,
! 3116: sqlite_int64 *piRowid, fts1Hash *pTerms){
! 3117: int rc;
! 3118:
! 3119: rc = content_insert(v, pRequestRowid, pValues); /* execute an SQL INSERT */
! 3120: if( rc!=SQLITE_OK ) return rc;
! 3121: *piRowid = sqlite3_last_insert_rowid(v->db);
! 3122: return insertTerms(v, pTerms, *piRowid, pValues);
! 3123: }
! 3124:
! 3125: /* Delete a row from the %_content table; fill [pTerms] with empty doclists
! 3126: * to be written to the %_term table. */
! 3127: static int index_delete(fulltext_vtab *v, sqlite_int64 iRow, fts1Hash *pTerms){
! 3128: int rc = deleteTerms(v, pTerms, iRow);
! 3129: if( rc!=SQLITE_OK ) return rc;
! 3130: return content_delete(v, iRow); /* execute an SQL DELETE */
! 3131: }
! 3132:
! 3133: /* Update a row in the %_content table; fill [pTerms] with new doclists for the
! 3134: * %_term table. */
! 3135: static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
! 3136: sqlite3_value **pValues, fts1Hash *pTerms){
! 3137: /* Generate an empty doclist for each term that previously appeared in this
! 3138: * row. */
! 3139: int rc = deleteTerms(v, pTerms, iRow);
! 3140: if( rc!=SQLITE_OK ) return rc;
! 3141:
! 3142: rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */
! 3143: if( rc!=SQLITE_OK ) return rc;
! 3144:
! 3145: /* Now add positions for terms which appear in the updated row. */
! 3146: return insertTerms(v, pTerms, iRow, pValues);
! 3147: }
! 3148:
! 3149: /* This function implements the xUpdate callback; it is the top-level entry
! 3150: * point for inserting, deleting or updating a row in a full-text table. */
! 3151: static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
! 3152: sqlite_int64 *pRowid){
! 3153: fulltext_vtab *v = (fulltext_vtab *) pVtab;
! 3154: fts1Hash terms; /* maps term string -> PosList */
! 3155: int rc;
! 3156: fts1HashElem *e;
! 3157:
! 3158: TRACE(("FTS1 Update %p\n", pVtab));
! 3159:
! 3160: fts1HashInit(&terms, FTS1_HASH_STRING, 1);
! 3161:
! 3162: if( nArg<2 ){
! 3163: rc = index_delete(v, sqlite3_value_int64(ppArg[0]), &terms);
! 3164: } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
! 3165: /* An update:
! 3166: * ppArg[0] = old rowid
! 3167: * ppArg[1] = new rowid
! 3168: * ppArg[2..2+v->nColumn-1] = values
! 3169: * ppArg[2+v->nColumn] = value for magic column (we ignore this)
! 3170: */
! 3171: sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
! 3172: if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER ||
! 3173: sqlite3_value_int64(ppArg[1]) != rowid ){
! 3174: rc = SQLITE_ERROR; /* we don't allow changing the rowid */
! 3175: } else {
! 3176: assert( nArg==2+v->nColumn+1);
! 3177: rc = index_update(v, rowid, &ppArg[2], &terms);
! 3178: }
! 3179: } else {
! 3180: /* An insert:
! 3181: * ppArg[1] = requested rowid
! 3182: * ppArg[2..2+v->nColumn-1] = values
! 3183: * ppArg[2+v->nColumn] = value for magic column (we ignore this)
! 3184: */
! 3185: assert( nArg==2+v->nColumn+1);
! 3186: rc = index_insert(v, ppArg[1], &ppArg[2], pRowid, &terms);
! 3187: }
! 3188:
! 3189: if( rc==SQLITE_OK ){
! 3190: /* Write updated doclists to disk. */
! 3191: for(e=fts1HashFirst(&terms); e; e=fts1HashNext(e)){
! 3192: DocList *p = fts1HashData(e);
! 3193: rc = index_insert_term(v, fts1HashKey(e), fts1HashKeysize(e), p);
! 3194: if( rc!=SQLITE_OK ) break;
! 3195: }
! 3196: }
! 3197:
! 3198: /* clean up */
! 3199: for(e=fts1HashFirst(&terms); e; e=fts1HashNext(e)){
! 3200: DocList *p = fts1HashData(e);
! 3201: docListDelete(p);
! 3202: }
! 3203: fts1HashClear(&terms);
! 3204:
! 3205: return rc;
! 3206: }
! 3207:
! 3208: /*
! 3209: ** Implementation of the snippet() function for FTS1
! 3210: */
! 3211: static void snippetFunc(
! 3212: sqlite3_context *pContext,
! 3213: int argc,
! 3214: sqlite3_value **argv
! 3215: ){
! 3216: fulltext_cursor *pCursor;
! 3217: if( argc<1 ) return;
! 3218: if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
! 3219: sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
! 3220: sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
! 3221: }else{
! 3222: const char *zStart = "<b>";
! 3223: const char *zEnd = "</b>";
! 3224: const char *zEllipsis = "<b>...</b>";
! 3225: memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
! 3226: if( argc>=2 ){
! 3227: zStart = (const char*)sqlite3_value_text(argv[1]);
! 3228: if( argc>=3 ){
! 3229: zEnd = (const char*)sqlite3_value_text(argv[2]);
! 3230: if( argc>=4 ){
! 3231: zEllipsis = (const char*)sqlite3_value_text(argv[3]);
! 3232: }
! 3233: }
! 3234: }
! 3235: snippetAllOffsets(pCursor);
! 3236: snippetText(pCursor, zStart, zEnd, zEllipsis);
! 3237: sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
! 3238: pCursor->snippet.nSnippet, SQLITE_STATIC);
! 3239: }
! 3240: }
! 3241:
! 3242: /*
! 3243: ** Implementation of the offsets() function for FTS1
! 3244: */
! 3245: static void snippetOffsetsFunc(
! 3246: sqlite3_context *pContext,
! 3247: int argc,
! 3248: sqlite3_value **argv
! 3249: ){
! 3250: fulltext_cursor *pCursor;
! 3251: if( argc<1 ) return;
! 3252: if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
! 3253: sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
! 3254: sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
! 3255: }else{
! 3256: memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
! 3257: snippetAllOffsets(pCursor);
! 3258: snippetOffsetText(&pCursor->snippet);
! 3259: sqlite3_result_text(pContext,
! 3260: pCursor->snippet.zOffset, pCursor->snippet.nOffset,
! 3261: SQLITE_STATIC);
! 3262: }
! 3263: }
! 3264:
! 3265: /*
! 3266: ** This routine implements the xFindFunction method for the FTS1
! 3267: ** virtual table.
! 3268: */
! 3269: static int fulltextFindFunction(
! 3270: sqlite3_vtab *pVtab,
! 3271: int nArg,
! 3272: const char *zName,
! 3273: void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
! 3274: void **ppArg
! 3275: ){
! 3276: if( strcmp(zName,"snippet")==0 ){
! 3277: *pxFunc = snippetFunc;
! 3278: return 1;
! 3279: }else if( strcmp(zName,"offsets")==0 ){
! 3280: *pxFunc = snippetOffsetsFunc;
! 3281: return 1;
! 3282: }
! 3283: return 0;
! 3284: }
! 3285:
! 3286: /*
! 3287: ** Rename an fts1 table.
! 3288: */
! 3289: static int fulltextRename(
! 3290: sqlite3_vtab *pVtab,
! 3291: const char *zName
! 3292: ){
! 3293: fulltext_vtab *p = (fulltext_vtab *)pVtab;
! 3294: int rc = SQLITE_NOMEM;
! 3295: char *zSql = sqlite3_mprintf(
! 3296: "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
! 3297: "ALTER TABLE %Q.'%q_term' RENAME TO '%q_term';"
! 3298: , p->zDb, p->zName, zName
! 3299: , p->zDb, p->zName, zName
! 3300: );
! 3301: if( zSql ){
! 3302: rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
! 3303: sqlite3_free(zSql);
! 3304: }
! 3305: return rc;
! 3306: }
! 3307:
! 3308: static const sqlite3_module fulltextModule = {
! 3309: /* iVersion */ 0,
! 3310: /* xCreate */ fulltextCreate,
! 3311: /* xConnect */ fulltextConnect,
! 3312: /* xBestIndex */ fulltextBestIndex,
! 3313: /* xDisconnect */ fulltextDisconnect,
! 3314: /* xDestroy */ fulltextDestroy,
! 3315: /* xOpen */ fulltextOpen,
! 3316: /* xClose */ fulltextClose,
! 3317: /* xFilter */ fulltextFilter,
! 3318: /* xNext */ fulltextNext,
! 3319: /* xEof */ fulltextEof,
! 3320: /* xColumn */ fulltextColumn,
! 3321: /* xRowid */ fulltextRowid,
! 3322: /* xUpdate */ fulltextUpdate,
! 3323: /* xBegin */ 0,
! 3324: /* xSync */ 0,
! 3325: /* xCommit */ 0,
! 3326: /* xRollback */ 0,
! 3327: /* xFindFunction */ fulltextFindFunction,
! 3328: /* xRename */ fulltextRename,
! 3329: };
! 3330:
! 3331: int sqlite3Fts1Init(sqlite3 *db){
! 3332: sqlite3_overload_function(db, "snippet", -1);
! 3333: sqlite3_overload_function(db, "offsets", -1);
! 3334: return sqlite3_create_module(db, "fts1", &fulltextModule, 0);
! 3335: }
! 3336:
! 3337: #if !SQLITE_CORE
! 3338: int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg,
! 3339: const sqlite3_api_routines *pApi){
! 3340: SQLITE_EXTENSION_INIT2(pApi)
! 3341: return sqlite3Fts1Init(db);
! 3342: }
! 3343: #endif
! 3344:
! 3345: #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts1.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts1