1: /* fts2 has a design flaw which can lead to database corruption (see
2: ** below). It is recommended not to use it any longer, instead use
3: ** fts3 (or higher). If you believe that your use of fts2 is safe,
4: ** add -DSQLITE_ENABLE_BROKEN_FTS2=1 to your CFLAGS.
5: */
6: #if (!defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)) \
7: && !defined(SQLITE_ENABLE_BROKEN_FTS2)
8: #error fts2 has a design flaw and has been deprecated.
9: #endif
10: /* The flaw is that fts2 uses the content table's unaliased rowid as
11: ** the unique docid. fts2 embeds the rowid in the index it builds,
12: ** and expects the rowid to not change. The SQLite VACUUM operation
13: ** will renumber such rowids, thereby breaking fts2. If you are using
14: ** fts2 in a system which has disabled VACUUM, then you can continue
15: ** to use it safely. Note that PRAGMA auto_vacuum does NOT disable
16: ** VACUUM, though systems using auto_vacuum are unlikely to invoke
17: ** VACUUM.
18: **
19: ** Unlike fts1, which is safe across VACUUM if you never delete
20: ** documents, fts2 has a second exposure to this flaw, in the segments
21: ** table. So fts2 should be considered unsafe across VACUUM in all
22: ** cases.
23: */
24:
25: /*
26: ** 2006 Oct 10
27: **
28: ** The author disclaims copyright to this source code. In place of
29: ** a legal notice, here is a blessing:
30: **
31: ** May you do good and not evil.
32: ** May you find forgiveness for yourself and forgive others.
33: ** May you share freely, never taking more than you give.
34: **
35: ******************************************************************************
36: **
37: ** This is an SQLite module implementing full-text search.
38: */
39:
40: /*
41: ** The code in this file is only compiled if:
42: **
43: ** * The FTS2 module is being built as an extension
44: ** (in which case SQLITE_CORE is not defined), or
45: **
46: ** * The FTS2 module is being built into the core of
47: ** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
48: */
49:
50: /* TODO(shess) Consider exporting this comment to an HTML file or the
51: ** wiki.
52: */
53: /* The full-text index is stored in a series of b+tree (-like)
54: ** structures called segments which map terms to doclists. The
55: ** structures are like b+trees in layout, but are constructed from the
56: ** bottom up in optimal fashion and are not updatable. Since trees
57: ** are built from the bottom up, things will be described from the
58: ** bottom up.
59: **
60: **
61: **** Varints ****
62: ** The basic unit of encoding is a variable-length integer called a
63: ** varint. We encode variable-length integers in little-endian order
64: ** using seven bits * per byte as follows:
65: **
66: ** KEY:
67: ** A = 0xxxxxxx 7 bits of data and one flag bit
68: ** B = 1xxxxxxx 7 bits of data and one flag bit
69: **
70: ** 7 bits - A
71: ** 14 bits - BA
72: ** 21 bits - BBA
73: ** and so on.
74: **
75: ** This is identical to how sqlite encodes varints (see util.c).
76: **
77: **
78: **** Document lists ****
79: ** A doclist (document list) holds a docid-sorted list of hits for a
80: ** given term. Doclists hold docids, and can optionally associate
81: ** token positions and offsets with docids.
82: **
83: ** A DL_POSITIONS_OFFSETS doclist is stored like this:
84: **
85: ** array {
86: ** varint docid;
87: ** array { (position list for column 0)
88: ** varint position; (delta from previous position plus POS_BASE)
89: ** varint startOffset; (delta from previous startOffset)
90: ** varint endOffset; (delta from startOffset)
91: ** }
92: ** array {
93: ** varint POS_COLUMN; (marks start of position list for new column)
94: ** varint column; (index of new column)
95: ** array {
96: ** varint position; (delta from previous position plus POS_BASE)
97: ** varint startOffset;(delta from previous startOffset)
98: ** varint endOffset; (delta from startOffset)
99: ** }
100: ** }
101: ** varint POS_END; (marks end of positions for this document.
102: ** }
103: **
104: ** Here, array { X } means zero or more occurrences of X, adjacent in
105: ** memory. A "position" is an index of a token in the token stream
106: ** generated by the tokenizer, while an "offset" is a byte offset,
107: ** both based at 0. Note that POS_END and POS_COLUMN occur in the
108: ** same logical place as the position element, and act as sentinals
109: ** ending a position list array.
110: **
111: ** A DL_POSITIONS doclist omits the startOffset and endOffset
112: ** information. A DL_DOCIDS doclist omits both the position and
113: ** offset information, becoming an array of varint-encoded docids.
114: **
115: ** On-disk data is stored as type DL_DEFAULT, so we don't serialize
116: ** the type. Due to how deletion is implemented in the segmentation
117: ** system, on-disk doclists MUST store at least positions.
118: **
119: **
120: **** Segment leaf nodes ****
121: ** Segment leaf nodes store terms and doclists, ordered by term. Leaf
122: ** nodes are written using LeafWriter, and read using LeafReader (to
123: ** iterate through a single leaf node's data) and LeavesReader (to
124: ** iterate through a segment's entire leaf layer). Leaf nodes have
125: ** the format:
126: **
127: ** varint iHeight; (height from leaf level, always 0)
128: ** varint nTerm; (length of first term)
129: ** char pTerm[nTerm]; (content of first term)
130: ** varint nDoclist; (length of term's associated doclist)
131: ** char pDoclist[nDoclist]; (content of doclist)
132: ** array {
133: ** (further terms are delta-encoded)
134: ** varint nPrefix; (length of prefix shared with previous term)
135: ** varint nSuffix; (length of unshared suffix)
136: ** char pTermSuffix[nSuffix];(unshared suffix of next term)
137: ** varint nDoclist; (length of term's associated doclist)
138: ** char pDoclist[nDoclist]; (content of doclist)
139: ** }
140: **
141: ** Here, array { X } means zero or more occurrences of X, adjacent in
142: ** memory.
143: **
144: ** Leaf nodes are broken into blocks which are stored contiguously in
145: ** the %_segments table in sorted order. This means that when the end
146: ** of a node is reached, the next term is in the node with the next
147: ** greater node id.
148: **
149: ** New data is spilled to a new leaf node when the current node
150: ** exceeds LEAF_MAX bytes (default 2048). New data which itself is
151: ** larger than STANDALONE_MIN (default 1024) is placed in a standalone
152: ** node (a leaf node with a single term and doclist). The goal of
153: ** these settings is to pack together groups of small doclists while
154: ** making it efficient to directly access large doclists. The
155: ** assumption is that large doclists represent terms which are more
156: ** likely to be query targets.
157: **
158: ** TODO(shess) It may be useful for blocking decisions to be more
159: ** dynamic. For instance, it may make more sense to have a 2.5k leaf
160: ** node rather than splitting into 2k and .5k nodes. My intuition is
161: ** that this might extend through 2x or 4x the pagesize.
162: **
163: **
164: **** Segment interior nodes ****
165: ** Segment interior nodes store blockids for subtree nodes and terms
166: ** to describe what data is stored by the each subtree. Interior
167: ** nodes are written using InteriorWriter, and read using
168: ** InteriorReader. InteriorWriters are created as needed when
169: ** SegmentWriter creates new leaf nodes, or when an interior node
170: ** itself grows too big and must be split. The format of interior
171: ** nodes:
172: **
173: ** varint iHeight; (height from leaf level, always >0)
174: ** varint iBlockid; (block id of node's leftmost subtree)
175: ** optional {
176: ** varint nTerm; (length of first term)
177: ** char pTerm[nTerm]; (content of first term)
178: ** array {
179: ** (further terms are delta-encoded)
180: ** varint nPrefix; (length of shared prefix with previous term)
181: ** varint nSuffix; (length of unshared suffix)
182: ** char pTermSuffix[nSuffix]; (unshared suffix of next term)
183: ** }
184: ** }
185: **
186: ** Here, optional { X } means an optional element, while array { X }
187: ** means zero or more occurrences of X, adjacent in memory.
188: **
189: ** An interior node encodes n terms separating n+1 subtrees. The
190: ** subtree blocks are contiguous, so only the first subtree's blockid
191: ** is encoded. The subtree at iBlockid will contain all terms less
192: ** than the first term encoded (or all terms if no term is encoded).
193: ** Otherwise, for terms greater than or equal to pTerm[i] but less
194: ** than pTerm[i+1], the subtree for that term will be rooted at
195: ** iBlockid+i. Interior nodes only store enough term data to
196: ** distinguish adjacent children (if the rightmost term of the left
197: ** child is "something", and the leftmost term of the right child is
198: ** "wicked", only "w" is stored).
199: **
200: ** New data is spilled to a new interior node at the same height when
201: ** the current node exceeds INTERIOR_MAX bytes (default 2048).
202: ** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing
203: ** interior nodes and making the tree too skinny. The interior nodes
204: ** at a given height are naturally tracked by interior nodes at
205: ** height+1, and so on.
206: **
207: **
208: **** Segment directory ****
209: ** The segment directory in table %_segdir stores meta-information for
210: ** merging and deleting segments, and also the root node of the
211: ** segment's tree.
212: **
213: ** The root node is the top node of the segment's tree after encoding
214: ** the entire segment, restricted to ROOT_MAX bytes (default 1024).
215: ** This could be either a leaf node or an interior node. If the top
216: ** node requires more than ROOT_MAX bytes, it is flushed to %_segments
217: ** and a new root interior node is generated (which should always fit
218: ** within ROOT_MAX because it only needs space for 2 varints, the
219: ** height and the blockid of the previous root).
220: **
221: ** The meta-information in the segment directory is:
222: ** level - segment level (see below)
223: ** idx - index within level
224: ** - (level,idx uniquely identify a segment)
225: ** start_block - first leaf node
226: ** leaves_end_block - last leaf node
227: ** end_block - last block (including interior nodes)
228: ** root - contents of root node
229: **
230: ** If the root node is a leaf node, then start_block,
231: ** leaves_end_block, and end_block are all 0.
232: **
233: **
234: **** Segment merging ****
235: ** To amortize update costs, segments are groups into levels and
236: ** merged in matches. Each increase in level represents exponentially
237: ** more documents.
238: **
239: ** New documents (actually, document updates) are tokenized and
240: ** written individually (using LeafWriter) to a level 0 segment, with
241: ** incrementing idx. When idx reaches MERGE_COUNT (default 16), all
242: ** level 0 segments are merged into a single level 1 segment. Level 1
243: ** is populated like level 0, and eventually MERGE_COUNT level 1
244: ** segments are merged to a single level 2 segment (representing
245: ** MERGE_COUNT^2 updates), and so on.
246: **
247: ** A segment merge traverses all segments at a given level in
248: ** parallel, performing a straightforward sorted merge. Since segment
249: ** leaf nodes are written in to the %_segments table in order, this
250: ** merge traverses the underlying sqlite disk structures efficiently.
251: ** After the merge, all segment blocks from the merged level are
252: ** deleted.
253: **
254: ** MERGE_COUNT controls how often we merge segments. 16 seems to be
255: ** somewhat of a sweet spot for insertion performance. 32 and 64 show
256: ** very similar performance numbers to 16 on insertion, though they're
257: ** a tiny bit slower (perhaps due to more overhead in merge-time
258: ** sorting). 8 is about 20% slower than 16, 4 about 50% slower than
259: ** 16, 2 about 66% slower than 16.
260: **
261: ** At query time, high MERGE_COUNT increases the number of segments
262: ** which need to be scanned and merged. For instance, with 100k docs
263: ** inserted:
264: **
265: ** MERGE_COUNT segments
266: ** 16 25
267: ** 8 12
268: ** 4 10
269: ** 2 6
270: **
271: ** This appears to have only a moderate impact on queries for very
272: ** frequent terms (which are somewhat dominated by segment merge
273: ** costs), and infrequent and non-existent terms still seem to be fast
274: ** even with many segments.
275: **
276: ** TODO(shess) That said, it would be nice to have a better query-side
277: ** argument for MERGE_COUNT of 16. Also, it is possible/likely that
278: ** optimizations to things like doclist merging will swing the sweet
279: ** spot around.
280: **
281: **
282: **
283: **** Handling of deletions and updates ****
284: ** Since we're using a segmented structure, with no docid-oriented
285: ** index into the term index, we clearly cannot simply update the term
286: ** index when a document is deleted or updated. For deletions, we
287: ** write an empty doclist (varint(docid) varint(POS_END)), for updates
288: ** we simply write the new doclist. Segment merges overwrite older
289: ** data for a particular docid with newer data, so deletes or updates
290: ** will eventually overtake the earlier data and knock it out. The
291: ** query logic likewise merges doclists so that newer data knocks out
292: ** older data.
293: **
294: ** TODO(shess) Provide a VACUUM type operation to clear out all
295: ** deletions and duplications. This would basically be a forced merge
296: ** into a single segment.
297: */
298:
299: #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
300:
301: #if defined(SQLITE_ENABLE_FTS2) && !defined(SQLITE_CORE)
302: # define SQLITE_CORE 1
303: #endif
304:
305: #include <assert.h>
306: #include <stdlib.h>
307: #include <stdio.h>
308: #include <string.h>
309: #include "fts2.h"
310: #include "fts2_hash.h"
311: #include "fts2_tokenizer.h"
312: #include "sqlite3.h"
313: #include "sqlite3ext.h"
314: SQLITE_EXTENSION_INIT1
315:
316:
317: /* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
318: ** would be nice to order the file better, perhaps something along the
319: ** lines of:
320: **
321: ** - utility functions
322: ** - table setup functions
323: ** - table update functions
324: ** - table query functions
325: **
326: ** Put the query functions last because they're likely to reference
327: ** typedefs or functions from the table update section.
328: */
329:
330: #if 0
331: # define TRACE(A) printf A; fflush(stdout)
332: #else
333: # define TRACE(A)
334: #endif
335:
336: /* It is not safe to call isspace(), tolower(), or isalnum() on
337: ** hi-bit-set characters. This is the same solution used in the
338: ** tokenizer.
339: */
340: /* TODO(shess) The snippet-generation code should be using the
341: ** tokenizer-generated tokens rather than doing its own local
342: ** tokenization.
343: */
344: /* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
345: static int safe_isspace(char c){
346: return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
347: }
348: static int safe_tolower(char c){
349: return (c>='A' && c<='Z') ? (c - 'A' + 'a') : c;
350: }
351: static int safe_isalnum(char c){
352: return (c>='0' && c<='9') || (c>='A' && c<='Z') || (c>='a' && c<='z');
353: }
354:
355: typedef enum DocListType {
356: DL_DOCIDS, /* docids only */
357: DL_POSITIONS, /* docids + positions */
358: DL_POSITIONS_OFFSETS /* docids + positions + offsets */
359: } DocListType;
360:
361: /*
362: ** By default, only positions and not offsets are stored in the doclists.
363: ** To change this so that offsets are stored too, compile with
364: **
365: ** -DDL_DEFAULT=DL_POSITIONS_OFFSETS
366: **
367: ** If DL_DEFAULT is set to DL_DOCIDS, your table can only be inserted
368: ** into (no deletes or updates).
369: */
370: #ifndef DL_DEFAULT
371: # define DL_DEFAULT DL_POSITIONS
372: #endif
373:
374: enum {
375: POS_END = 0, /* end of this position list */
376: POS_COLUMN, /* followed by new column number */
377: POS_BASE
378: };
379:
380: /* MERGE_COUNT controls how often we merge segments (see comment at
381: ** top of file).
382: */
383: #define MERGE_COUNT 16
384:
385: /* utility functions */
386:
387: /* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
388: ** record to prevent errors of the form:
389: **
390: ** my_function(SomeType *b){
391: ** memset(b, '\0', sizeof(b)); // sizeof(b)!=sizeof(*b)
392: ** }
393: */
394: /* TODO(shess) Obvious candidates for a header file. */
395: #define CLEAR(b) memset(b, '\0', sizeof(*(b)))
396:
397: #ifndef NDEBUG
398: # define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b)))
399: #else
400: # define SCRAMBLE(b)
401: #endif
402:
403: /* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
404: #define VARINT_MAX 10
405:
406: /* Write a 64-bit variable-length integer to memory starting at p[0].
407: * The length of data written will be between 1 and VARINT_MAX bytes.
408: * The number of bytes written is returned. */
409: static int putVarint(char *p, sqlite_int64 v){
410: unsigned char *q = (unsigned char *) p;
411: sqlite_uint64 vu = v;
412: do{
413: *q++ = (unsigned char) ((vu & 0x7f) | 0x80);
414: vu >>= 7;
415: }while( vu!=0 );
416: q[-1] &= 0x7f; /* turn off high bit in final byte */
417: assert( q - (unsigned char *)p <= VARINT_MAX );
418: return (int) (q - (unsigned char *)p);
419: }
420:
421: /* Read a 64-bit variable-length integer from memory starting at p[0].
422: * Return the number of bytes read, or 0 on error.
423: * The value is stored in *v. */
424: static int getVarint(const char *p, sqlite_int64 *v){
425: const unsigned char *q = (const unsigned char *) p;
426: sqlite_uint64 x = 0, y = 1;
427: while( (*q & 0x80) == 0x80 ){
428: x += y * (*q++ & 0x7f);
429: y <<= 7;
430: if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */
431: assert( 0 );
432: return 0;
433: }
434: }
435: x += y * (*q++);
436: *v = (sqlite_int64) x;
437: return (int) (q - (unsigned char *)p);
438: }
439:
440: static int getVarint32(const char *p, int *pi){
441: sqlite_int64 i;
442: int ret = getVarint(p, &i);
443: *pi = (int) i;
444: assert( *pi==i );
445: return ret;
446: }
447:
448: /*******************************************************************/
449: /* DataBuffer is used to collect data into a buffer in piecemeal
450: ** fashion. It implements the usual distinction between amount of
451: ** data currently stored (nData) and buffer capacity (nCapacity).
452: **
453: ** dataBufferInit - create a buffer with given initial capacity.
454: ** dataBufferReset - forget buffer's data, retaining capacity.
455: ** dataBufferDestroy - free buffer's data.
456: ** dataBufferSwap - swap contents of two buffers.
457: ** dataBufferExpand - expand capacity without adding data.
458: ** dataBufferAppend - append data.
459: ** dataBufferAppend2 - append two pieces of data at once.
460: ** dataBufferReplace - replace buffer's data.
461: */
462: typedef struct DataBuffer {
463: char *pData; /* Pointer to malloc'ed buffer. */
464: int nCapacity; /* Size of pData buffer. */
465: int nData; /* End of data loaded into pData. */
466: } DataBuffer;
467:
468: static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
469: assert( nCapacity>=0 );
470: pBuffer->nData = 0;
471: pBuffer->nCapacity = nCapacity;
472: pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
473: }
474: static void dataBufferReset(DataBuffer *pBuffer){
475: pBuffer->nData = 0;
476: }
477: static void dataBufferDestroy(DataBuffer *pBuffer){
478: if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData);
479: SCRAMBLE(pBuffer);
480: }
481: static void dataBufferSwap(DataBuffer *pBuffer1, DataBuffer *pBuffer2){
482: DataBuffer tmp = *pBuffer1;
483: *pBuffer1 = *pBuffer2;
484: *pBuffer2 = tmp;
485: }
486: static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
487: assert( nAddCapacity>0 );
488: /* TODO(shess) Consider expanding more aggressively. Note that the
489: ** underlying malloc implementation may take care of such things for
490: ** us already.
491: */
492: if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
493: pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
494: pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
495: }
496: }
497: static void dataBufferAppend(DataBuffer *pBuffer,
498: const char *pSource, int nSource){
499: assert( nSource>0 && pSource!=NULL );
500: dataBufferExpand(pBuffer, nSource);
501: memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
502: pBuffer->nData += nSource;
503: }
504: static void dataBufferAppend2(DataBuffer *pBuffer,
505: const char *pSource1, int nSource1,
506: const char *pSource2, int nSource2){
507: assert( nSource1>0 && pSource1!=NULL );
508: assert( nSource2>0 && pSource2!=NULL );
509: dataBufferExpand(pBuffer, nSource1+nSource2);
510: memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
511: memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
512: pBuffer->nData += nSource1+nSource2;
513: }
514: static void dataBufferReplace(DataBuffer *pBuffer,
515: const char *pSource, int nSource){
516: dataBufferReset(pBuffer);
517: dataBufferAppend(pBuffer, pSource, nSource);
518: }
519:
520: /* StringBuffer is a null-terminated version of DataBuffer. */
521: typedef struct StringBuffer {
522: DataBuffer b; /* Includes null terminator. */
523: } StringBuffer;
524:
525: static void initStringBuffer(StringBuffer *sb){
526: dataBufferInit(&sb->b, 100);
527: dataBufferReplace(&sb->b, "", 1);
528: }
529: static int stringBufferLength(StringBuffer *sb){
530: return sb->b.nData-1;
531: }
532: static char *stringBufferData(StringBuffer *sb){
533: return sb->b.pData;
534: }
535: static void stringBufferDestroy(StringBuffer *sb){
536: dataBufferDestroy(&sb->b);
537: }
538:
539: static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){
540: assert( sb->b.nData>0 );
541: if( nFrom>0 ){
542: sb->b.nData--;
543: dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
544: }
545: }
546: static void append(StringBuffer *sb, const char *zFrom){
547: nappend(sb, zFrom, strlen(zFrom));
548: }
549:
550: /* Append a list of strings separated by commas. */
551: static void appendList(StringBuffer *sb, int nString, char **azString){
552: int i;
553: for(i=0; i<nString; ++i){
554: if( i>0 ) append(sb, ", ");
555: append(sb, azString[i]);
556: }
557: }
558:
559: static int endsInWhiteSpace(StringBuffer *p){
560: return stringBufferLength(p)>0 &&
561: safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
562: }
563:
564: /* If the StringBuffer ends in something other than white space, add a
565: ** single space character to the end.
566: */
567: static void appendWhiteSpace(StringBuffer *p){
568: if( stringBufferLength(p)==0 ) return;
569: if( !endsInWhiteSpace(p) ) append(p, " ");
570: }
571:
572: /* Remove white space from the end of the StringBuffer */
573: static void trimWhiteSpace(StringBuffer *p){
574: while( endsInWhiteSpace(p) ){
575: p->b.pData[--p->b.nData-1] = '\0';
576: }
577: }
578:
579: /*******************************************************************/
580: /* DLReader is used to read document elements from a doclist. The
581: ** current docid is cached, so dlrDocid() is fast. DLReader does not
582: ** own the doclist buffer.
583: **
584: ** dlrAtEnd - true if there's no more data to read.
585: ** dlrDocid - docid of current document.
586: ** dlrDocData - doclist data for current document (including docid).
587: ** dlrDocDataBytes - length of same.
588: ** dlrAllDataBytes - length of all remaining data.
589: ** dlrPosData - position data for current document.
590: ** dlrPosDataLen - length of pos data for current document (incl POS_END).
591: ** dlrStep - step to current document.
592: ** dlrInit - initial for doclist of given type against given data.
593: ** dlrDestroy - clean up.
594: **
595: ** Expected usage is something like:
596: **
597: ** DLReader reader;
598: ** dlrInit(&reader, pData, nData);
599: ** while( !dlrAtEnd(&reader) ){
600: ** // calls to dlrDocid() and kin.
601: ** dlrStep(&reader);
602: ** }
603: ** dlrDestroy(&reader);
604: */
605: typedef struct DLReader {
606: DocListType iType;
607: const char *pData;
608: int nData;
609:
610: sqlite_int64 iDocid;
611: int nElement;
612: } DLReader;
613:
614: static int dlrAtEnd(DLReader *pReader){
615: assert( pReader->nData>=0 );
616: return pReader->nData==0;
617: }
618: static sqlite_int64 dlrDocid(DLReader *pReader){
619: assert( !dlrAtEnd(pReader) );
620: return pReader->iDocid;
621: }
622: static const char *dlrDocData(DLReader *pReader){
623: assert( !dlrAtEnd(pReader) );
624: return pReader->pData;
625: }
626: static int dlrDocDataBytes(DLReader *pReader){
627: assert( !dlrAtEnd(pReader) );
628: return pReader->nElement;
629: }
630: static int dlrAllDataBytes(DLReader *pReader){
631: assert( !dlrAtEnd(pReader) );
632: return pReader->nData;
633: }
634: /* TODO(shess) Consider adding a field to track iDocid varint length
635: ** to make these two functions faster. This might matter (a tiny bit)
636: ** for queries.
637: */
638: static const char *dlrPosData(DLReader *pReader){
639: sqlite_int64 iDummy;
640: int n = getVarint(pReader->pData, &iDummy);
641: assert( !dlrAtEnd(pReader) );
642: return pReader->pData+n;
643: }
644: static int dlrPosDataLen(DLReader *pReader){
645: sqlite_int64 iDummy;
646: int n = getVarint(pReader->pData, &iDummy);
647: assert( !dlrAtEnd(pReader) );
648: return pReader->nElement-n;
649: }
650: static void dlrStep(DLReader *pReader){
651: assert( !dlrAtEnd(pReader) );
652:
653: /* Skip past current doclist element. */
654: assert( pReader->nElement<=pReader->nData );
655: pReader->pData += pReader->nElement;
656: pReader->nData -= pReader->nElement;
657:
658: /* If there is more data, read the next doclist element. */
659: if( pReader->nData!=0 ){
660: sqlite_int64 iDocidDelta;
661: int iDummy, n = getVarint(pReader->pData, &iDocidDelta);
662: pReader->iDocid += iDocidDelta;
663: if( pReader->iType>=DL_POSITIONS ){
664: assert( n<pReader->nData );
665: while( 1 ){
666: n += getVarint32(pReader->pData+n, &iDummy);
667: assert( n<=pReader->nData );
668: if( iDummy==POS_END ) break;
669: if( iDummy==POS_COLUMN ){
670: n += getVarint32(pReader->pData+n, &iDummy);
671: assert( n<pReader->nData );
672: }else if( pReader->iType==DL_POSITIONS_OFFSETS ){
673: n += getVarint32(pReader->pData+n, &iDummy);
674: n += getVarint32(pReader->pData+n, &iDummy);
675: assert( n<pReader->nData );
676: }
677: }
678: }
679: pReader->nElement = n;
680: assert( pReader->nElement<=pReader->nData );
681: }
682: }
683: static void dlrInit(DLReader *pReader, DocListType iType,
684: const char *pData, int nData){
685: assert( pData!=NULL && nData!=0 );
686: pReader->iType = iType;
687: pReader->pData = pData;
688: pReader->nData = nData;
689: pReader->nElement = 0;
690: pReader->iDocid = 0;
691:
692: /* Load the first element's data. There must be a first element. */
693: dlrStep(pReader);
694: }
695: static void dlrDestroy(DLReader *pReader){
696: SCRAMBLE(pReader);
697: }
698:
699: #ifndef NDEBUG
700: /* Verify that the doclist can be validly decoded. Also returns the
701: ** last docid found because it is convenient in other assertions for
702: ** DLWriter.
703: */
704: static void docListValidate(DocListType iType, const char *pData, int nData,
705: sqlite_int64 *pLastDocid){
706: sqlite_int64 iPrevDocid = 0;
707: assert( nData>0 );
708: assert( pData!=0 );
709: assert( pData+nData>pData );
710: while( nData!=0 ){
711: sqlite_int64 iDocidDelta;
712: int n = getVarint(pData, &iDocidDelta);
713: iPrevDocid += iDocidDelta;
714: if( iType>DL_DOCIDS ){
715: int iDummy;
716: while( 1 ){
717: n += getVarint32(pData+n, &iDummy);
718: if( iDummy==POS_END ) break;
719: if( iDummy==POS_COLUMN ){
720: n += getVarint32(pData+n, &iDummy);
721: }else if( iType>DL_POSITIONS ){
722: n += getVarint32(pData+n, &iDummy);
723: n += getVarint32(pData+n, &iDummy);
724: }
725: assert( n<=nData );
726: }
727: }
728: assert( n<=nData );
729: pData += n;
730: nData -= n;
731: }
732: if( pLastDocid ) *pLastDocid = iPrevDocid;
733: }
734: #define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o)
735: #else
736: #define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 )
737: #endif
738:
739: /*******************************************************************/
740: /* DLWriter is used to write doclist data to a DataBuffer. DLWriter
741: ** always appends to the buffer and does not own it.
742: **
743: ** dlwInit - initialize to write a given type doclistto a buffer.
744: ** dlwDestroy - clear the writer's memory. Does not free buffer.
745: ** dlwAppend - append raw doclist data to buffer.
746: ** dlwCopy - copy next doclist from reader to writer.
747: ** dlwAdd - construct doclist element and append to buffer.
748: ** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter).
749: */
750: typedef struct DLWriter {
751: DocListType iType;
752: DataBuffer *b;
753: sqlite_int64 iPrevDocid;
754: #ifndef NDEBUG
755: int has_iPrevDocid;
756: #endif
757: } DLWriter;
758:
759: static void dlwInit(DLWriter *pWriter, DocListType iType, DataBuffer *b){
760: pWriter->b = b;
761: pWriter->iType = iType;
762: pWriter->iPrevDocid = 0;
763: #ifndef NDEBUG
764: pWriter->has_iPrevDocid = 0;
765: #endif
766: }
767: static void dlwDestroy(DLWriter *pWriter){
768: SCRAMBLE(pWriter);
769: }
770: /* iFirstDocid is the first docid in the doclist in pData. It is
771: ** needed because pData may point within a larger doclist, in which
772: ** case the first item would be delta-encoded.
773: **
774: ** iLastDocid is the final docid in the doclist in pData. It is
775: ** needed to create the new iPrevDocid for future delta-encoding. The
776: ** code could decode the passed doclist to recreate iLastDocid, but
777: ** the only current user (docListMerge) already has decoded this
778: ** information.
779: */
780: /* TODO(shess) This has become just a helper for docListMerge.
781: ** Consider a refactor to make this cleaner.
782: */
783: static void dlwAppend(DLWriter *pWriter,
784: const char *pData, int nData,
785: sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
786: sqlite_int64 iDocid = 0;
787: char c[VARINT_MAX];
788: int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */
789: #ifndef NDEBUG
790: sqlite_int64 iLastDocidDelta;
791: #endif
792:
793: /* Recode the initial docid as delta from iPrevDocid. */
794: nFirstOld = getVarint(pData, &iDocid);
795: assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
796: nFirstNew = putVarint(c, iFirstDocid-pWriter->iPrevDocid);
797:
798: /* Verify that the incoming doclist is valid AND that it ends with
799: ** the expected docid. This is essential because we'll trust this
800: ** docid in future delta-encoding.
801: */
802: ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta);
803: assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta );
804:
805: /* Append recoded initial docid and everything else. Rest of docids
806: ** should have been delta-encoded from previous initial docid.
807: */
808: if( nFirstOld<nData ){
809: dataBufferAppend2(pWriter->b, c, nFirstNew,
810: pData+nFirstOld, nData-nFirstOld);
811: }else{
812: dataBufferAppend(pWriter->b, c, nFirstNew);
813: }
814: pWriter->iPrevDocid = iLastDocid;
815: }
816: static void dlwCopy(DLWriter *pWriter, DLReader *pReader){
817: dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
818: dlrDocid(pReader), dlrDocid(pReader));
819: }
820: static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
821: char c[VARINT_MAX];
822: int n = putVarint(c, iDocid-pWriter->iPrevDocid);
823:
824: /* Docids must ascend. */
825: assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid );
826: assert( pWriter->iType==DL_DOCIDS );
827:
828: dataBufferAppend(pWriter->b, c, n);
829: pWriter->iPrevDocid = iDocid;
830: #ifndef NDEBUG
831: pWriter->has_iPrevDocid = 1;
832: #endif
833: }
834:
835: /*******************************************************************/
836: /* PLReader is used to read data from a document's position list. As
837: ** the caller steps through the list, data is cached so that varints
838: ** only need to be decoded once.
839: **
840: ** plrInit, plrDestroy - create/destroy a reader.
841: ** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors
842: ** plrAtEnd - at end of stream, only call plrDestroy once true.
843: ** plrStep - step to the next element.
844: */
845: typedef struct PLReader {
846: /* These refer to the next position's data. nData will reach 0 when
847: ** reading the last position, so plrStep() signals EOF by setting
848: ** pData to NULL.
849: */
850: const char *pData;
851: int nData;
852:
853: DocListType iType;
854: int iColumn; /* the last column read */
855: int iPosition; /* the last position read */
856: int iStartOffset; /* the last start offset read */
857: int iEndOffset; /* the last end offset read */
858: } PLReader;
859:
860: static int plrAtEnd(PLReader *pReader){
861: return pReader->pData==NULL;
862: }
863: static int plrColumn(PLReader *pReader){
864: assert( !plrAtEnd(pReader) );
865: return pReader->iColumn;
866: }
867: static int plrPosition(PLReader *pReader){
868: assert( !plrAtEnd(pReader) );
869: return pReader->iPosition;
870: }
871: static int plrStartOffset(PLReader *pReader){
872: assert( !plrAtEnd(pReader) );
873: return pReader->iStartOffset;
874: }
875: static int plrEndOffset(PLReader *pReader){
876: assert( !plrAtEnd(pReader) );
877: return pReader->iEndOffset;
878: }
879: static void plrStep(PLReader *pReader){
880: int i, n;
881:
882: assert( !plrAtEnd(pReader) );
883:
884: if( pReader->nData==0 ){
885: pReader->pData = NULL;
886: return;
887: }
888:
889: n = getVarint32(pReader->pData, &i);
890: if( i==POS_COLUMN ){
891: n += getVarint32(pReader->pData+n, &pReader->iColumn);
892: pReader->iPosition = 0;
893: pReader->iStartOffset = 0;
894: n += getVarint32(pReader->pData+n, &i);
895: }
896: /* Should never see adjacent column changes. */
897: assert( i!=POS_COLUMN );
898:
899: if( i==POS_END ){
900: pReader->nData = 0;
901: pReader->pData = NULL;
902: return;
903: }
904:
905: pReader->iPosition += i-POS_BASE;
906: if( pReader->iType==DL_POSITIONS_OFFSETS ){
907: n += getVarint32(pReader->pData+n, &i);
908: pReader->iStartOffset += i;
909: n += getVarint32(pReader->pData+n, &i);
910: pReader->iEndOffset = pReader->iStartOffset+i;
911: }
912: assert( n<=pReader->nData );
913: pReader->pData += n;
914: pReader->nData -= n;
915: }
916:
917: static void plrInit(PLReader *pReader, DLReader *pDLReader){
918: pReader->pData = dlrPosData(pDLReader);
919: pReader->nData = dlrPosDataLen(pDLReader);
920: pReader->iType = pDLReader->iType;
921: pReader->iColumn = 0;
922: pReader->iPosition = 0;
923: pReader->iStartOffset = 0;
924: pReader->iEndOffset = 0;
925: plrStep(pReader);
926: }
927: static void plrDestroy(PLReader *pReader){
928: SCRAMBLE(pReader);
929: }
930:
931: /*******************************************************************/
932: /* PLWriter is used in constructing a document's position list. As a
933: ** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op.
934: ** PLWriter writes to the associated DLWriter's buffer.
935: **
936: ** plwInit - init for writing a document's poslist.
937: ** plwDestroy - clear a writer.
938: ** plwAdd - append position and offset information.
939: ** plwCopy - copy next position's data from reader to writer.
940: ** plwTerminate - add any necessary doclist terminator.
941: **
942: ** Calling plwAdd() after plwTerminate() may result in a corrupt
943: ** doclist.
944: */
945: /* TODO(shess) Until we've written the second item, we can cache the
946: ** first item's information. Then we'd have three states:
947: **
948: ** - initialized with docid, no positions.
949: ** - docid and one position.
950: ** - docid and multiple positions.
951: **
952: ** Only the last state needs to actually write to dlw->b, which would
953: ** be an improvement in the DLCollector case.
954: */
955: typedef struct PLWriter {
956: DLWriter *dlw;
957:
958: int iColumn; /* the last column written */
959: int iPos; /* the last position written */
960: int iOffset; /* the last start offset written */
961: } PLWriter;
962:
963: /* TODO(shess) In the case where the parent is reading these values
964: ** from a PLReader, we could optimize to a copy if that PLReader has
965: ** the same type as pWriter.
966: */
967: static void plwAdd(PLWriter *pWriter, int iColumn, int iPos,
968: int iStartOffset, int iEndOffset){
969: /* Worst-case space for POS_COLUMN, iColumn, iPosDelta,
970: ** iStartOffsetDelta, and iEndOffsetDelta.
971: */
972: char c[5*VARINT_MAX];
973: int n = 0;
974:
975: /* Ban plwAdd() after plwTerminate(). */
976: assert( pWriter->iPos!=-1 );
977:
978: if( pWriter->dlw->iType==DL_DOCIDS ) return;
979:
980: if( iColumn!=pWriter->iColumn ){
981: n += putVarint(c+n, POS_COLUMN);
982: n += putVarint(c+n, iColumn);
983: pWriter->iColumn = iColumn;
984: pWriter->iPos = 0;
985: pWriter->iOffset = 0;
986: }
987: assert( iPos>=pWriter->iPos );
988: n += putVarint(c+n, POS_BASE+(iPos-pWriter->iPos));
989: pWriter->iPos = iPos;
990: if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){
991: assert( iStartOffset>=pWriter->iOffset );
992: n += putVarint(c+n, iStartOffset-pWriter->iOffset);
993: pWriter->iOffset = iStartOffset;
994: assert( iEndOffset>=iStartOffset );
995: n += putVarint(c+n, iEndOffset-iStartOffset);
996: }
997: dataBufferAppend(pWriter->dlw->b, c, n);
998: }
999: static void plwCopy(PLWriter *pWriter, PLReader *pReader){
1000: plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader),
1001: plrStartOffset(pReader), plrEndOffset(pReader));
1002: }
1003: static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){
1004: char c[VARINT_MAX];
1005: int n;
1006:
1007: pWriter->dlw = dlw;
1008:
1009: /* Docids must ascend. */
1010: assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid );
1011: n = putVarint(c, iDocid-pWriter->dlw->iPrevDocid);
1012: dataBufferAppend(pWriter->dlw->b, c, n);
1013: pWriter->dlw->iPrevDocid = iDocid;
1014: #ifndef NDEBUG
1015: pWriter->dlw->has_iPrevDocid = 1;
1016: #endif
1017:
1018: pWriter->iColumn = 0;
1019: pWriter->iPos = 0;
1020: pWriter->iOffset = 0;
1021: }
1022: /* TODO(shess) Should plwDestroy() also terminate the doclist? But
1023: ** then plwDestroy() would no longer be just a destructor, it would
1024: ** also be doing work, which isn't consistent with the overall idiom.
1025: ** Another option would be for plwAdd() to always append any necessary
1026: ** terminator, so that the output is always correct. But that would
1027: ** add incremental work to the common case with the only benefit being
1028: ** API elegance. Punt for now.
1029: */
1030: static void plwTerminate(PLWriter *pWriter){
1031: if( pWriter->dlw->iType>DL_DOCIDS ){
1032: char c[VARINT_MAX];
1033: int n = putVarint(c, POS_END);
1034: dataBufferAppend(pWriter->dlw->b, c, n);
1035: }
1036: #ifndef NDEBUG
1037: /* Mark as terminated for assert in plwAdd(). */
1038: pWriter->iPos = -1;
1039: #endif
1040: }
1041: static void plwDestroy(PLWriter *pWriter){
1042: SCRAMBLE(pWriter);
1043: }
1044:
1045: /*******************************************************************/
1046: /* DLCollector wraps PLWriter and DLWriter to provide a
1047: ** dynamically-allocated doclist area to use during tokenization.
1048: **
1049: ** dlcNew - malloc up and initialize a collector.
1050: ** dlcDelete - destroy a collector and all contained items.
1051: ** dlcAddPos - append position and offset information.
1052: ** dlcAddDoclist - add the collected doclist to the given buffer.
1053: ** dlcNext - terminate the current document and open another.
1054: */
1055: typedef struct DLCollector {
1056: DataBuffer b;
1057: DLWriter dlw;
1058: PLWriter plw;
1059: } DLCollector;
1060:
1061: /* TODO(shess) This could also be done by calling plwTerminate() and
1062: ** dataBufferAppend(). I tried that, expecting nominal performance
1063: ** differences, but it seemed to pretty reliably be worth 1% to code
1064: ** it this way. I suspect it is the incremental malloc overhead (some
1065: ** percentage of the plwTerminate() calls will cause a realloc), so
1066: ** this might be worth revisiting if the DataBuffer implementation
1067: ** changes.
1068: */
1069: static void dlcAddDoclist(DLCollector *pCollector, DataBuffer *b){
1070: if( pCollector->dlw.iType>DL_DOCIDS ){
1071: char c[VARINT_MAX];
1072: int n = putVarint(c, POS_END);
1073: dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
1074: }else{
1075: dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
1076: }
1077: }
1078: static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
1079: plwTerminate(&pCollector->plw);
1080: plwDestroy(&pCollector->plw);
1081: plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
1082: }
1083: static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
1084: int iStartOffset, int iEndOffset){
1085: plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
1086: }
1087:
1088: static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){
1089: DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
1090: dataBufferInit(&pCollector->b, 0);
1091: dlwInit(&pCollector->dlw, iType, &pCollector->b);
1092: plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
1093: return pCollector;
1094: }
1095: static void dlcDelete(DLCollector *pCollector){
1096: plwDestroy(&pCollector->plw);
1097: dlwDestroy(&pCollector->dlw);
1098: dataBufferDestroy(&pCollector->b);
1099: SCRAMBLE(pCollector);
1100: sqlite3_free(pCollector);
1101: }
1102:
1103:
1104: /* Copy the doclist data of iType in pData/nData into *out, trimming
1105: ** unnecessary data as we go. Only columns matching iColumn are
1106: ** copied, all columns copied if iColumn is -1. Elements with no
1107: ** matching columns are dropped. The output is an iOutType doclist.
1108: */
1109: /* NOTE(shess) This code is only valid after all doclists are merged.
1110: ** If this is run before merges, then doclist items which represent
1111: ** deletion will be trimmed, and will thus not effect a deletion
1112: ** during the merge.
1113: */
1114: static void docListTrim(DocListType iType, const char *pData, int nData,
1115: int iColumn, DocListType iOutType, DataBuffer *out){
1116: DLReader dlReader;
1117: DLWriter dlWriter;
1118:
1119: assert( iOutType<=iType );
1120:
1121: dlrInit(&dlReader, iType, pData, nData);
1122: dlwInit(&dlWriter, iOutType, out);
1123:
1124: while( !dlrAtEnd(&dlReader) ){
1125: PLReader plReader;
1126: PLWriter plWriter;
1127: int match = 0;
1128:
1129: plrInit(&plReader, &dlReader);
1130:
1131: while( !plrAtEnd(&plReader) ){
1132: if( iColumn==-1 || plrColumn(&plReader)==iColumn ){
1133: if( !match ){
1134: plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader));
1135: match = 1;
1136: }
1137: plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
1138: plrStartOffset(&plReader), plrEndOffset(&plReader));
1139: }
1140: plrStep(&plReader);
1141: }
1142: if( match ){
1143: plwTerminate(&plWriter);
1144: plwDestroy(&plWriter);
1145: }
1146:
1147: plrDestroy(&plReader);
1148: dlrStep(&dlReader);
1149: }
1150: dlwDestroy(&dlWriter);
1151: dlrDestroy(&dlReader);
1152: }
1153:
1154: /* Used by docListMerge() to keep doclists in the ascending order by
1155: ** docid, then ascending order by age (so the newest comes first).
1156: */
1157: typedef struct OrderedDLReader {
1158: DLReader *pReader;
1159:
1160: /* TODO(shess) If we assume that docListMerge pReaders is ordered by
1161: ** age (which we do), then we could use pReader comparisons to break
1162: ** ties.
1163: */
1164: int idx;
1165: } OrderedDLReader;
1166:
1167: /* Order eof to end, then by docid asc, idx desc. */
1168: static int orderedDLReaderCmp(OrderedDLReader *r1, OrderedDLReader *r2){
1169: if( dlrAtEnd(r1->pReader) ){
1170: if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */
1171: return 1; /* Only r1 atEnd(). */
1172: }
1173: if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */
1174:
1175: if( dlrDocid(r1->pReader)<dlrDocid(r2->pReader) ) return -1;
1176: if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1;
1177:
1178: /* Descending on idx. */
1179: return r2->idx-r1->idx;
1180: }
1181:
1182: /* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that
1183: ** p[1..n-1] is already sorted.
1184: */
1185: /* TODO(shess) Is this frequent enough to warrant a binary search?
1186: ** Before implementing that, instrument the code to check. In most
1187: ** current usage, I expect that p[0] will be less than p[1] a very
1188: ** high proportion of the time.
1189: */
1190: static void orderedDLReaderReorder(OrderedDLReader *p, int n){
1191: while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){
1192: OrderedDLReader tmp = p[0];
1193: p[0] = p[1];
1194: p[1] = tmp;
1195: n--;
1196: p++;
1197: }
1198: }
1199:
1200: /* Given an array of doclist readers, merge their doclist elements
1201: ** into out in sorted order (by docid), dropping elements from older
1202: ** readers when there is a duplicate docid. pReaders is assumed to be
1203: ** ordered by age, oldest first.
1204: */
1205: /* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably
1206: ** be fixed.
1207: */
1208: static void docListMerge(DataBuffer *out,
1209: DLReader *pReaders, int nReaders){
1210: OrderedDLReader readers[MERGE_COUNT];
1211: DLWriter writer;
1212: int i, n;
1213: const char *pStart = 0;
1214: int nStart = 0;
1215: sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
1216:
1217: assert( nReaders>0 );
1218: if( nReaders==1 ){
1219: dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
1220: return;
1221: }
1222:
1223: assert( nReaders<=MERGE_COUNT );
1224: n = 0;
1225: for(i=0; i<nReaders; i++){
1226: assert( pReaders[i].iType==pReaders[0].iType );
1227: readers[i].pReader = pReaders+i;
1228: readers[i].idx = i;
1229: n += dlrAllDataBytes(&pReaders[i]);
1230: }
1231: /* Conservatively size output to sum of inputs. Output should end
1232: ** up strictly smaller than input.
1233: */
1234: dataBufferExpand(out, n);
1235:
1236: /* Get the readers into sorted order. */
1237: while( i-->0 ){
1238: orderedDLReaderReorder(readers+i, nReaders-i);
1239: }
1240:
1241: dlwInit(&writer, pReaders[0].iType, out);
1242: while( !dlrAtEnd(readers[0].pReader) ){
1243: sqlite_int64 iDocid = dlrDocid(readers[0].pReader);
1244:
1245: /* If this is a continuation of the current buffer to copy, extend
1246: ** that buffer. memcpy() seems to be more efficient if it has a
1247: ** lots of data to copy.
1248: */
1249: if( dlrDocData(readers[0].pReader)==pStart+nStart ){
1250: nStart += dlrDocDataBytes(readers[0].pReader);
1251: }else{
1252: if( pStart!=0 ){
1253: dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
1254: }
1255: pStart = dlrDocData(readers[0].pReader);
1256: nStart = dlrDocDataBytes(readers[0].pReader);
1257: iFirstDocid = iDocid;
1258: }
1259: iLastDocid = iDocid;
1260: dlrStep(readers[0].pReader);
1261:
1262: /* Drop all of the older elements with the same docid. */
1263: for(i=1; i<nReaders &&
1264: !dlrAtEnd(readers[i].pReader) &&
1265: dlrDocid(readers[i].pReader)==iDocid; i++){
1266: dlrStep(readers[i].pReader);
1267: }
1268:
1269: /* Get the readers back into order. */
1270: while( i-->0 ){
1271: orderedDLReaderReorder(readers+i, nReaders-i);
1272: }
1273: }
1274:
1275: /* Copy over any remaining elements. */
1276: if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
1277: dlwDestroy(&writer);
1278: }
1279:
1280: /* Helper function for posListUnion(). Compares the current position
1281: ** between left and right, returning as standard C idiom of <0 if
1282: ** left<right, >0 if left>right, and 0 if left==right. "End" always
1283: ** compares greater.
1284: */
1285: static int posListCmp(PLReader *pLeft, PLReader *pRight){
1286: assert( pLeft->iType==pRight->iType );
1287: if( pLeft->iType==DL_DOCIDS ) return 0;
1288:
1289: if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1;
1290: if( plrAtEnd(pRight) ) return -1;
1291:
1292: if( plrColumn(pLeft)<plrColumn(pRight) ) return -1;
1293: if( plrColumn(pLeft)>plrColumn(pRight) ) return 1;
1294:
1295: if( plrPosition(pLeft)<plrPosition(pRight) ) return -1;
1296: if( plrPosition(pLeft)>plrPosition(pRight) ) return 1;
1297: if( pLeft->iType==DL_POSITIONS ) return 0;
1298:
1299: if( plrStartOffset(pLeft)<plrStartOffset(pRight) ) return -1;
1300: if( plrStartOffset(pLeft)>plrStartOffset(pRight) ) return 1;
1301:
1302: if( plrEndOffset(pLeft)<plrEndOffset(pRight) ) return -1;
1303: if( plrEndOffset(pLeft)>plrEndOffset(pRight) ) return 1;
1304:
1305: return 0;
1306: }
1307:
1308: /* Write the union of position lists in pLeft and pRight to pOut.
1309: ** "Union" in this case meaning "All unique position tuples". Should
1310: ** work with any doclist type, though both inputs and the output
1311: ** should be the same type.
1312: */
1313: static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
1314: PLReader left, right;
1315: PLWriter writer;
1316:
1317: assert( dlrDocid(pLeft)==dlrDocid(pRight) );
1318: assert( pLeft->iType==pRight->iType );
1319: assert( pLeft->iType==pOut->iType );
1320:
1321: plrInit(&left, pLeft);
1322: plrInit(&right, pRight);
1323: plwInit(&writer, pOut, dlrDocid(pLeft));
1324:
1325: while( !plrAtEnd(&left) || !plrAtEnd(&right) ){
1326: int c = posListCmp(&left, &right);
1327: if( c<0 ){
1328: plwCopy(&writer, &left);
1329: plrStep(&left);
1330: }else if( c>0 ){
1331: plwCopy(&writer, &right);
1332: plrStep(&right);
1333: }else{
1334: plwCopy(&writer, &left);
1335: plrStep(&left);
1336: plrStep(&right);
1337: }
1338: }
1339:
1340: plwTerminate(&writer);
1341: plwDestroy(&writer);
1342: plrDestroy(&left);
1343: plrDestroy(&right);
1344: }
1345:
1346: /* Write the union of doclists in pLeft and pRight to pOut. For
1347: ** docids in common between the inputs, the union of the position
1348: ** lists is written. Inputs and outputs are always type DL_DEFAULT.
1349: */
1350: static void docListUnion(
1351: const char *pLeft, int nLeft,
1352: const char *pRight, int nRight,
1353: DataBuffer *pOut /* Write the combined doclist here */
1354: ){
1355: DLReader left, right;
1356: DLWriter writer;
1357:
1358: if( nLeft==0 ){
1359: if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
1360: return;
1361: }
1362: if( nRight==0 ){
1363: dataBufferAppend(pOut, pLeft, nLeft);
1364: return;
1365: }
1366:
1367: dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
1368: dlrInit(&right, DL_DEFAULT, pRight, nRight);
1369: dlwInit(&writer, DL_DEFAULT, pOut);
1370:
1371: while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
1372: if( dlrAtEnd(&right) ){
1373: dlwCopy(&writer, &left);
1374: dlrStep(&left);
1375: }else if( dlrAtEnd(&left) ){
1376: dlwCopy(&writer, &right);
1377: dlrStep(&right);
1378: }else if( dlrDocid(&left)<dlrDocid(&right) ){
1379: dlwCopy(&writer, &left);
1380: dlrStep(&left);
1381: }else if( dlrDocid(&left)>dlrDocid(&right) ){
1382: dlwCopy(&writer, &right);
1383: dlrStep(&right);
1384: }else{
1385: posListUnion(&left, &right, &writer);
1386: dlrStep(&left);
1387: dlrStep(&right);
1388: }
1389: }
1390:
1391: dlrDestroy(&left);
1392: dlrDestroy(&right);
1393: dlwDestroy(&writer);
1394: }
1395:
1396: /* pLeft and pRight are DLReaders positioned to the same docid.
1397: **
1398: ** If there are no instances in pLeft or pRight where the position
1399: ** of pLeft is one less than the position of pRight, then this
1400: ** routine adds nothing to pOut.
1401: **
1402: ** If there are one or more instances where positions from pLeft
1403: ** are exactly one less than positions from pRight, then add a new
1404: ** document record to pOut. If pOut wants to hold positions, then
1405: ** include the positions from pRight that are one more than a
1406: ** position in pLeft. In other words: pRight.iPos==pLeft.iPos+1.
1407: */
1408: static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight,
1409: DLWriter *pOut){
1410: PLReader left, right;
1411: PLWriter writer;
1412: int match = 0;
1413:
1414: assert( dlrDocid(pLeft)==dlrDocid(pRight) );
1415: assert( pOut->iType!=DL_POSITIONS_OFFSETS );
1416:
1417: plrInit(&left, pLeft);
1418: plrInit(&right, pRight);
1419:
1420: while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
1421: if( plrColumn(&left)<plrColumn(&right) ){
1422: plrStep(&left);
1423: }else if( plrColumn(&left)>plrColumn(&right) ){
1424: plrStep(&right);
1425: }else if( plrPosition(&left)+1<plrPosition(&right) ){
1426: plrStep(&left);
1427: }else if( plrPosition(&left)+1>plrPosition(&right) ){
1428: plrStep(&right);
1429: }else{
1430: if( !match ){
1431: plwInit(&writer, pOut, dlrDocid(pLeft));
1432: match = 1;
1433: }
1434: plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
1435: plrStep(&left);
1436: plrStep(&right);
1437: }
1438: }
1439:
1440: if( match ){
1441: plwTerminate(&writer);
1442: plwDestroy(&writer);
1443: }
1444:
1445: plrDestroy(&left);
1446: plrDestroy(&right);
1447: }
1448:
1449: /* We have two doclists with positions: pLeft and pRight.
1450: ** Write the phrase intersection of these two doclists into pOut.
1451: **
1452: ** A phrase intersection means that two documents only match
1453: ** if pLeft.iPos+1==pRight.iPos.
1454: **
1455: ** iType controls the type of data written to pOut. If iType is
1456: ** DL_POSITIONS, the positions are those from pRight.
1457: */
1458: static void docListPhraseMerge(
1459: const char *pLeft, int nLeft,
1460: const char *pRight, int nRight,
1461: DocListType iType,
1462: DataBuffer *pOut /* Write the combined doclist here */
1463: ){
1464: DLReader left, right;
1465: DLWriter writer;
1466:
1467: if( nLeft==0 || nRight==0 ) return;
1468:
1469: assert( iType!=DL_POSITIONS_OFFSETS );
1470:
1471: dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
1472: dlrInit(&right, DL_POSITIONS, pRight, nRight);
1473: dlwInit(&writer, iType, pOut);
1474:
1475: while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
1476: if( dlrDocid(&left)<dlrDocid(&right) ){
1477: dlrStep(&left);
1478: }else if( dlrDocid(&right)<dlrDocid(&left) ){
1479: dlrStep(&right);
1480: }else{
1481: posListPhraseMerge(&left, &right, &writer);
1482: dlrStep(&left);
1483: dlrStep(&right);
1484: }
1485: }
1486:
1487: dlrDestroy(&left);
1488: dlrDestroy(&right);
1489: dlwDestroy(&writer);
1490: }
1491:
1492: /* We have two DL_DOCIDS doclists: pLeft and pRight.
1493: ** Write the intersection of these two doclists into pOut as a
1494: ** DL_DOCIDS doclist.
1495: */
1496: static void docListAndMerge(
1497: const char *pLeft, int nLeft,
1498: const char *pRight, int nRight,
1499: DataBuffer *pOut /* Write the combined doclist here */
1500: ){
1501: DLReader left, right;
1502: DLWriter writer;
1503:
1504: if( nLeft==0 || nRight==0 ) return;
1505:
1506: dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
1507: dlrInit(&right, DL_DOCIDS, pRight, nRight);
1508: dlwInit(&writer, DL_DOCIDS, pOut);
1509:
1510: while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
1511: if( dlrDocid(&left)<dlrDocid(&right) ){
1512: dlrStep(&left);
1513: }else if( dlrDocid(&right)<dlrDocid(&left) ){
1514: dlrStep(&right);
1515: }else{
1516: dlwAdd(&writer, dlrDocid(&left));
1517: dlrStep(&left);
1518: dlrStep(&right);
1519: }
1520: }
1521:
1522: dlrDestroy(&left);
1523: dlrDestroy(&right);
1524: dlwDestroy(&writer);
1525: }
1526:
1527: /* We have two DL_DOCIDS doclists: pLeft and pRight.
1528: ** Write the union of these two doclists into pOut as a
1529: ** DL_DOCIDS doclist.
1530: */
1531: static void docListOrMerge(
1532: const char *pLeft, int nLeft,
1533: const char *pRight, int nRight,
1534: DataBuffer *pOut /* Write the combined doclist here */
1535: ){
1536: DLReader left, right;
1537: DLWriter writer;
1538:
1539: if( nLeft==0 ){
1540: if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
1541: return;
1542: }
1543: if( nRight==0 ){
1544: dataBufferAppend(pOut, pLeft, nLeft);
1545: return;
1546: }
1547:
1548: dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
1549: dlrInit(&right, DL_DOCIDS, pRight, nRight);
1550: dlwInit(&writer, DL_DOCIDS, pOut);
1551:
1552: while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
1553: if( dlrAtEnd(&right) ){
1554: dlwAdd(&writer, dlrDocid(&left));
1555: dlrStep(&left);
1556: }else if( dlrAtEnd(&left) ){
1557: dlwAdd(&writer, dlrDocid(&right));
1558: dlrStep(&right);
1559: }else if( dlrDocid(&left)<dlrDocid(&right) ){
1560: dlwAdd(&writer, dlrDocid(&left));
1561: dlrStep(&left);
1562: }else if( dlrDocid(&right)<dlrDocid(&left) ){
1563: dlwAdd(&writer, dlrDocid(&right));
1564: dlrStep(&right);
1565: }else{
1566: dlwAdd(&writer, dlrDocid(&left));
1567: dlrStep(&left);
1568: dlrStep(&right);
1569: }
1570: }
1571:
1572: dlrDestroy(&left);
1573: dlrDestroy(&right);
1574: dlwDestroy(&writer);
1575: }
1576:
1577: /* We have two DL_DOCIDS doclists: pLeft and pRight.
1578: ** Write into pOut as DL_DOCIDS doclist containing all documents that
1579: ** occur in pLeft but not in pRight.
1580: */
1581: static void docListExceptMerge(
1582: const char *pLeft, int nLeft,
1583: const char *pRight, int nRight,
1584: DataBuffer *pOut /* Write the combined doclist here */
1585: ){
1586: DLReader left, right;
1587: DLWriter writer;
1588:
1589: if( nLeft==0 ) return;
1590: if( nRight==0 ){
1591: dataBufferAppend(pOut, pLeft, nLeft);
1592: return;
1593: }
1594:
1595: dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
1596: dlrInit(&right, DL_DOCIDS, pRight, nRight);
1597: dlwInit(&writer, DL_DOCIDS, pOut);
1598:
1599: while( !dlrAtEnd(&left) ){
1600: while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
1601: dlrStep(&right);
1602: }
1603: if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){
1604: dlwAdd(&writer, dlrDocid(&left));
1605: }
1606: dlrStep(&left);
1607: }
1608:
1609: dlrDestroy(&left);
1610: dlrDestroy(&right);
1611: dlwDestroy(&writer);
1612: }
1613:
1614: static char *string_dup_n(const char *s, int n){
1615: char *str = sqlite3_malloc(n + 1);
1616: memcpy(str, s, n);
1617: str[n] = '\0';
1618: return str;
1619: }
1620:
1621: /* Duplicate a string; the caller must free() the returned string.
1622: * (We don't use strdup() since it is not part of the standard C library and
1623: * may not be available everywhere.) */
1624: static char *string_dup(const char *s){
1625: return string_dup_n(s, strlen(s));
1626: }
1627:
1628: /* Format a string, replacing each occurrence of the % character with
1629: * zDb.zName. This may be more convenient than sqlite_mprintf()
1630: * when one string is used repeatedly in a format string.
1631: * The caller must free() the returned string. */
1632: static char *string_format(const char *zFormat,
1633: const char *zDb, const char *zName){
1634: const char *p;
1635: size_t len = 0;
1636: size_t nDb = strlen(zDb);
1637: size_t nName = strlen(zName);
1638: size_t nFullTableName = nDb+1+nName;
1639: char *result;
1640: char *r;
1641:
1642: /* first compute length needed */
1643: for(p = zFormat ; *p ; ++p){
1644: len += (*p=='%' ? nFullTableName : 1);
1645: }
1646: len += 1; /* for null terminator */
1647:
1648: r = result = sqlite3_malloc(len);
1649: for(p = zFormat; *p; ++p){
1650: if( *p=='%' ){
1651: memcpy(r, zDb, nDb);
1652: r += nDb;
1653: *r++ = '.';
1654: memcpy(r, zName, nName);
1655: r += nName;
1656: } else {
1657: *r++ = *p;
1658: }
1659: }
1660: *r++ = '\0';
1661: assert( r == result + len );
1662: return result;
1663: }
1664:
1665: static int sql_exec(sqlite3 *db, const char *zDb, const char *zName,
1666: const char *zFormat){
1667: char *zCommand = string_format(zFormat, zDb, zName);
1668: int rc;
1669: TRACE(("FTS2 sql: %s\n", zCommand));
1670: rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
1671: sqlite3_free(zCommand);
1672: return rc;
1673: }
1674:
1675: static int sql_prepare(sqlite3 *db, const char *zDb, const char *zName,
1676: sqlite3_stmt **ppStmt, const char *zFormat){
1677: char *zCommand = string_format(zFormat, zDb, zName);
1678: int rc;
1679: TRACE(("FTS2 prepare: %s\n", zCommand));
1680: rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL);
1681: sqlite3_free(zCommand);
1682: return rc;
1683: }
1684:
1685: /* end utility functions */
1686:
1687: /* Forward reference */
1688: typedef struct fulltext_vtab fulltext_vtab;
1689:
1690: /* A single term in a query is represented by an instances of
1691: ** the following structure.
1692: */
1693: typedef struct QueryTerm {
1694: short int nPhrase; /* How many following terms are part of the same phrase */
1695: short int iPhrase; /* This is the i-th term of a phrase. */
1696: short int iColumn; /* Column of the index that must match this term */
1697: signed char isOr; /* this term is preceded by "OR" */
1698: signed char isNot; /* this term is preceded by "-" */
1699: signed char isPrefix; /* this term is followed by "*" */
1700: char *pTerm; /* text of the term. '\000' terminated. malloced */
1701: int nTerm; /* Number of bytes in pTerm[] */
1702: } QueryTerm;
1703:
1704:
1705: /* A query string is parsed into a Query structure.
1706: *
1707: * We could, in theory, allow query strings to be complicated
1708: * nested expressions with precedence determined by parentheses.
1709: * But none of the major search engines do this. (Perhaps the
1710: * feeling is that an parenthesized expression is two complex of
1711: * an idea for the average user to grasp.) Taking our lead from
1712: * the major search engines, we will allow queries to be a list
1713: * of terms (with an implied AND operator) or phrases in double-quotes,
1714: * with a single optional "-" before each non-phrase term to designate
1715: * negation and an optional OR connector.
1716: *
1717: * OR binds more tightly than the implied AND, which is what the
1718: * major search engines seem to do. So, for example:
1719: *
1720: * [one two OR three] ==> one AND (two OR three)
1721: * [one OR two three] ==> (one OR two) AND three
1722: *
1723: * A "-" before a term matches all entries that lack that term.
1724: * The "-" must occur immediately before the term with in intervening
1725: * space. This is how the search engines do it.
1726: *
1727: * A NOT term cannot be the right-hand operand of an OR. If this
1728: * occurs in the query string, the NOT is ignored:
1729: *
1730: * [one OR -two] ==> one OR two
1731: *
1732: */
1733: typedef struct Query {
1734: fulltext_vtab *pFts; /* The full text index */
1735: int nTerms; /* Number of terms in the query */
1736: QueryTerm *pTerms; /* Array of terms. Space obtained from malloc() */
1737: int nextIsOr; /* Set the isOr flag on the next inserted term */
1738: int nextColumn; /* Next word parsed must be in this column */
1739: int dfltColumn; /* The default column */
1740: } Query;
1741:
1742:
1743: /*
1744: ** An instance of the following structure keeps track of generated
1745: ** matching-word offset information and snippets.
1746: */
1747: typedef struct Snippet {
1748: int nMatch; /* Total number of matches */
1749: int nAlloc; /* Space allocated for aMatch[] */
1750: struct snippetMatch { /* One entry for each matching term */
1751: char snStatus; /* Status flag for use while constructing snippets */
1752: short int iCol; /* The column that contains the match */
1753: short int iTerm; /* The index in Query.pTerms[] of the matching term */
1754: short int nByte; /* Number of bytes in the term */
1755: int iStart; /* The offset to the first character of the term */
1756: } *aMatch; /* Points to space obtained from malloc */
1757: char *zOffset; /* Text rendering of aMatch[] */
1758: int nOffset; /* strlen(zOffset) */
1759: char *zSnippet; /* Snippet text */
1760: int nSnippet; /* strlen(zSnippet) */
1761: } Snippet;
1762:
1763:
1764: typedef enum QueryType {
1765: QUERY_GENERIC, /* table scan */
1766: QUERY_ROWID, /* lookup by rowid */
1767: QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
1768: } QueryType;
1769:
1770: typedef enum fulltext_statement {
1771: CONTENT_INSERT_STMT,
1772: CONTENT_SELECT_STMT,
1773: CONTENT_UPDATE_STMT,
1774: CONTENT_DELETE_STMT,
1775: CONTENT_EXISTS_STMT,
1776:
1777: BLOCK_INSERT_STMT,
1778: BLOCK_SELECT_STMT,
1779: BLOCK_DELETE_STMT,
1780: BLOCK_DELETE_ALL_STMT,
1781:
1782: SEGDIR_MAX_INDEX_STMT,
1783: SEGDIR_SET_STMT,
1784: SEGDIR_SELECT_LEVEL_STMT,
1785: SEGDIR_SPAN_STMT,
1786: SEGDIR_DELETE_STMT,
1787: SEGDIR_SELECT_SEGMENT_STMT,
1788: SEGDIR_SELECT_ALL_STMT,
1789: SEGDIR_DELETE_ALL_STMT,
1790: SEGDIR_COUNT_STMT,
1791:
1792: MAX_STMT /* Always at end! */
1793: } fulltext_statement;
1794:
1795: /* These must exactly match the enum above. */
1796: /* TODO(shess): Is there some risk that a statement will be used in two
1797: ** cursors at once, e.g. if a query joins a virtual table to itself?
1798: ** If so perhaps we should move some of these to the cursor object.
1799: */
1800: static const char *const fulltext_zStatement[MAX_STMT] = {
1801: /* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */
1802: /* CONTENT_SELECT */ "select * from %_content where rowid = ?",
1803: /* CONTENT_UPDATE */ NULL, /* generated in contentUpdateStatement() */
1804: /* CONTENT_DELETE */ "delete from %_content where rowid = ?",
1805: /* CONTENT_EXISTS */ "select rowid from %_content limit 1",
1806:
1807: /* BLOCK_INSERT */ "insert into %_segments values (?)",
1808: /* BLOCK_SELECT */ "select block from %_segments where rowid = ?",
1809: /* BLOCK_DELETE */ "delete from %_segments where rowid between ? and ?",
1810: /* BLOCK_DELETE_ALL */ "delete from %_segments",
1811:
1812: /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
1813: /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
1814: /* SEGDIR_SELECT_LEVEL */
1815: "select start_block, leaves_end_block, root from %_segdir "
1816: " where level = ? order by idx",
1817: /* SEGDIR_SPAN */
1818: "select min(start_block), max(end_block) from %_segdir "
1819: " where level = ? and start_block <> 0",
1820: /* SEGDIR_DELETE */ "delete from %_segdir where level = ?",
1821:
1822: /* NOTE(shess): The first three results of the following two
1823: ** statements must match.
1824: */
1825: /* SEGDIR_SELECT_SEGMENT */
1826: "select start_block, leaves_end_block, root from %_segdir "
1827: " where level = ? and idx = ?",
1828: /* SEGDIR_SELECT_ALL */
1829: "select start_block, leaves_end_block, root from %_segdir "
1830: " order by level desc, idx asc",
1831: /* SEGDIR_DELETE_ALL */ "delete from %_segdir",
1832: /* SEGDIR_COUNT */ "select count(*), ifnull(max(level),0) from %_segdir",
1833: };
1834:
1835: /*
1836: ** A connection to a fulltext index is an instance of the following
1837: ** structure. The xCreate and xConnect methods create an instance
1838: ** of this structure and xDestroy and xDisconnect free that instance.
1839: ** All other methods receive a pointer to the structure as one of their
1840: ** arguments.
1841: */
1842: struct fulltext_vtab {
1843: sqlite3_vtab base; /* Base class used by SQLite core */
1844: sqlite3 *db; /* The database connection */
1845: const char *zDb; /* logical database name */
1846: const char *zName; /* virtual table name */
1847: int nColumn; /* number of columns in virtual table */
1848: char **azColumn; /* column names. malloced */
1849: char **azContentColumn; /* column names in content table; malloced */
1850: sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
1851:
1852: /* Precompiled statements which we keep as long as the table is
1853: ** open.
1854: */
1855: sqlite3_stmt *pFulltextStatements[MAX_STMT];
1856:
1857: /* Precompiled statements used for segment merges. We run a
1858: ** separate select across the leaf level of each tree being merged.
1859: */
1860: sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
1861: /* The statement used to prepare pLeafSelectStmts. */
1862: #define LEAF_SELECT \
1863: "select block from %_segments where rowid between ? and ? order by rowid"
1864:
1865: /* These buffer pending index updates during transactions.
1866: ** nPendingData estimates the memory size of the pending data. It
1867: ** doesn't include the hash-bucket overhead, nor any malloc
1868: ** overhead. When nPendingData exceeds kPendingThreshold, the
1869: ** buffer is flushed even before the transaction closes.
1870: ** pendingTerms stores the data, and is only valid when nPendingData
1871: ** is >=0 (nPendingData<0 means pendingTerms has not been
1872: ** initialized). iPrevDocid is the last docid written, used to make
1873: ** certain we're inserting in sorted order.
1874: */
1875: int nPendingData;
1876: #define kPendingThreshold (1*1024*1024)
1877: sqlite_int64 iPrevDocid;
1878: fts2Hash pendingTerms;
1879: };
1880:
1881: /*
1882: ** When the core wants to do a query, it create a cursor using a
1883: ** call to xOpen. This structure is an instance of a cursor. It
1884: ** is destroyed by xClose.
1885: */
1886: typedef struct fulltext_cursor {
1887: sqlite3_vtab_cursor base; /* Base class used by SQLite core */
1888: QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */
1889: sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
1890: int eof; /* True if at End Of Results */
1891: Query q; /* Parsed query string */
1892: Snippet snippet; /* Cached snippet for the current row */
1893: int iColumn; /* Column being searched */
1894: DataBuffer result; /* Doclist results from fulltextQuery */
1895: DLReader reader; /* Result reader if result not empty */
1896: } fulltext_cursor;
1897:
1898: static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){
1899: return (fulltext_vtab *) c->base.pVtab;
1900: }
1901:
1902: static const sqlite3_module fts2Module; /* forward declaration */
1903:
1904: /* Return a dynamically generated statement of the form
1905: * insert into %_content (rowid, ...) values (?, ...)
1906: */
1907: static const char *contentInsertStatement(fulltext_vtab *v){
1908: StringBuffer sb;
1909: int i;
1910:
1911: initStringBuffer(&sb);
1912: append(&sb, "insert into %_content (rowid, ");
1913: appendList(&sb, v->nColumn, v->azContentColumn);
1914: append(&sb, ") values (?");
1915: for(i=0; i<v->nColumn; ++i)
1916: append(&sb, ", ?");
1917: append(&sb, ")");
1918: return stringBufferData(&sb);
1919: }
1920:
1921: /* Return a dynamically generated statement of the form
1922: * update %_content set [col_0] = ?, [col_1] = ?, ...
1923: * where rowid = ?
1924: */
1925: static const char *contentUpdateStatement(fulltext_vtab *v){
1926: StringBuffer sb;
1927: int i;
1928:
1929: initStringBuffer(&sb);
1930: append(&sb, "update %_content set ");
1931: for(i=0; i<v->nColumn; ++i) {
1932: if( i>0 ){
1933: append(&sb, ", ");
1934: }
1935: append(&sb, v->azContentColumn[i]);
1936: append(&sb, " = ?");
1937: }
1938: append(&sb, " where rowid = ?");
1939: return stringBufferData(&sb);
1940: }
1941:
1942: /* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
1943: ** If the indicated statement has never been prepared, it is prepared
1944: ** and cached, otherwise the cached version is reset.
1945: */
1946: static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
1947: sqlite3_stmt **ppStmt){
1948: assert( iStmt<MAX_STMT );
1949: if( v->pFulltextStatements[iStmt]==NULL ){
1950: const char *zStmt;
1951: int rc;
1952: switch( iStmt ){
1953: case CONTENT_INSERT_STMT:
1954: zStmt = contentInsertStatement(v); break;
1955: case CONTENT_UPDATE_STMT:
1956: zStmt = contentUpdateStatement(v); break;
1957: default:
1958: zStmt = fulltext_zStatement[iStmt];
1959: }
1960: rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
1961: zStmt);
1962: if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt);
1963: if( rc!=SQLITE_OK ) return rc;
1964: } else {
1965: int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
1966: if( rc!=SQLITE_OK ) return rc;
1967: }
1968:
1969: *ppStmt = v->pFulltextStatements[iStmt];
1970: return SQLITE_OK;
1971: }
1972:
1973: /* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and
1974: ** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE,
1975: ** where we expect no results.
1976: */
1977: static int sql_single_step(sqlite3_stmt *s){
1978: int rc = sqlite3_step(s);
1979: return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
1980: }
1981:
1982: /* Like sql_get_statement(), but for special replicated LEAF_SELECT
1983: ** statements. idx -1 is a special case for an uncached version of
1984: ** the statement (used in the optimize implementation).
1985: */
1986: /* TODO(shess) Write version for generic statements and then share
1987: ** that between the cached-statement functions.
1988: */
1989: static int sql_get_leaf_statement(fulltext_vtab *v, int idx,
1990: sqlite3_stmt **ppStmt){
1991: assert( idx>=-1 && idx<MERGE_COUNT );
1992: if( idx==-1 ){
1993: return sql_prepare(v->db, v->zDb, v->zName, ppStmt, LEAF_SELECT);
1994: }else if( v->pLeafSelectStmts[idx]==NULL ){
1995: int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx],
1996: LEAF_SELECT);
1997: if( rc!=SQLITE_OK ) return rc;
1998: }else{
1999: int rc = sqlite3_reset(v->pLeafSelectStmts[idx]);
2000: if( rc!=SQLITE_OK ) return rc;
2001: }
2002:
2003: *ppStmt = v->pLeafSelectStmts[idx];
2004: return SQLITE_OK;
2005: }
2006:
2007: /* insert into %_content (rowid, ...) values ([rowid], [pValues]) */
2008: static int content_insert(fulltext_vtab *v, sqlite3_value *rowid,
2009: sqlite3_value **pValues){
2010: sqlite3_stmt *s;
2011: int i;
2012: int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
2013: if( rc!=SQLITE_OK ) return rc;
2014:
2015: rc = sqlite3_bind_value(s, 1, rowid);
2016: if( rc!=SQLITE_OK ) return rc;
2017:
2018: for(i=0; i<v->nColumn; ++i){
2019: rc = sqlite3_bind_value(s, 2+i, pValues[i]);
2020: if( rc!=SQLITE_OK ) return rc;
2021: }
2022:
2023: return sql_single_step(s);
2024: }
2025:
2026: /* update %_content set col0 = pValues[0], col1 = pValues[1], ...
2027: * where rowid = [iRowid] */
2028: static int content_update(fulltext_vtab *v, sqlite3_value **pValues,
2029: sqlite_int64 iRowid){
2030: sqlite3_stmt *s;
2031: int i;
2032: int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s);
2033: if( rc!=SQLITE_OK ) return rc;
2034:
2035: for(i=0; i<v->nColumn; ++i){
2036: rc = sqlite3_bind_value(s, 1+i, pValues[i]);
2037: if( rc!=SQLITE_OK ) return rc;
2038: }
2039:
2040: rc = sqlite3_bind_int64(s, 1+v->nColumn, iRowid);
2041: if( rc!=SQLITE_OK ) return rc;
2042:
2043: return sql_single_step(s);
2044: }
2045:
2046: static void freeStringArray(int nString, const char **pString){
2047: int i;
2048:
2049: for (i=0 ; i < nString ; ++i) {
2050: if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]);
2051: }
2052: sqlite3_free((void *) pString);
2053: }
2054:
2055: /* select * from %_content where rowid = [iRow]
2056: * The caller must delete the returned array and all strings in it.
2057: * null fields will be NULL in the returned array.
2058: *
2059: * TODO: Perhaps we should return pointer/length strings here for consistency
2060: * with other code which uses pointer/length. */
2061: static int content_select(fulltext_vtab *v, sqlite_int64 iRow,
2062: const char ***pValues){
2063: sqlite3_stmt *s;
2064: const char **values;
2065: int i;
2066: int rc;
2067:
2068: *pValues = NULL;
2069:
2070: rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
2071: if( rc!=SQLITE_OK ) return rc;
2072:
2073: rc = sqlite3_bind_int64(s, 1, iRow);
2074: if( rc!=SQLITE_OK ) return rc;
2075:
2076: rc = sqlite3_step(s);
2077: if( rc!=SQLITE_ROW ) return rc;
2078:
2079: values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *));
2080: for(i=0; i<v->nColumn; ++i){
2081: if( sqlite3_column_type(s, i)==SQLITE_NULL ){
2082: values[i] = NULL;
2083: }else{
2084: values[i] = string_dup((char*)sqlite3_column_text(s, i));
2085: }
2086: }
2087:
2088: /* We expect only one row. We must execute another sqlite3_step()
2089: * to complete the iteration; otherwise the table will remain locked. */
2090: rc = sqlite3_step(s);
2091: if( rc==SQLITE_DONE ){
2092: *pValues = values;
2093: return SQLITE_OK;
2094: }
2095:
2096: freeStringArray(v->nColumn, values);
2097: return rc;
2098: }
2099:
2100: /* delete from %_content where rowid = [iRow ] */
2101: static int content_delete(fulltext_vtab *v, sqlite_int64 iRow){
2102: sqlite3_stmt *s;
2103: int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
2104: if( rc!=SQLITE_OK ) return rc;
2105:
2106: rc = sqlite3_bind_int64(s, 1, iRow);
2107: if( rc!=SQLITE_OK ) return rc;
2108:
2109: return sql_single_step(s);
2110: }
2111:
2112: /* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if
2113: ** no rows exist, and any error in case of failure.
2114: */
2115: static int content_exists(fulltext_vtab *v){
2116: sqlite3_stmt *s;
2117: int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s);
2118: if( rc!=SQLITE_OK ) return rc;
2119:
2120: rc = sqlite3_step(s);
2121: if( rc!=SQLITE_ROW ) return rc;
2122:
2123: /* We expect only one row. We must execute another sqlite3_step()
2124: * to complete the iteration; otherwise the table will remain locked. */
2125: rc = sqlite3_step(s);
2126: if( rc==SQLITE_DONE ) return SQLITE_ROW;
2127: if( rc==SQLITE_ROW ) return SQLITE_ERROR;
2128: return rc;
2129: }
2130:
2131: /* insert into %_segments values ([pData])
2132: ** returns assigned rowid in *piBlockid
2133: */
2134: static int block_insert(fulltext_vtab *v, const char *pData, int nData,
2135: sqlite_int64 *piBlockid){
2136: sqlite3_stmt *s;
2137: int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s);
2138: if( rc!=SQLITE_OK ) return rc;
2139:
2140: rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC);
2141: if( rc!=SQLITE_OK ) return rc;
2142:
2143: rc = sqlite3_step(s);
2144: if( rc==SQLITE_ROW ) return SQLITE_ERROR;
2145: if( rc!=SQLITE_DONE ) return rc;
2146:
2147: *piBlockid = sqlite3_last_insert_rowid(v->db);
2148: return SQLITE_OK;
2149: }
2150:
2151: /* delete from %_segments
2152: ** where rowid between [iStartBlockid] and [iEndBlockid]
2153: **
2154: ** Deletes the range of blocks, inclusive, used to delete the blocks
2155: ** which form a segment.
2156: */
2157: static int block_delete(fulltext_vtab *v,
2158: sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){
2159: sqlite3_stmt *s;
2160: int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s);
2161: if( rc!=SQLITE_OK ) return rc;
2162:
2163: rc = sqlite3_bind_int64(s, 1, iStartBlockid);
2164: if( rc!=SQLITE_OK ) return rc;
2165:
2166: rc = sqlite3_bind_int64(s, 2, iEndBlockid);
2167: if( rc!=SQLITE_OK ) return rc;
2168:
2169: return sql_single_step(s);
2170: }
2171:
2172: /* Returns SQLITE_ROW with *pidx set to the maximum segment idx found
2173: ** at iLevel. Returns SQLITE_DONE if there are no segments at
2174: ** iLevel. Otherwise returns an error.
2175: */
2176: static int segdir_max_index(fulltext_vtab *v, int iLevel, int *pidx){
2177: sqlite3_stmt *s;
2178: int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s);
2179: if( rc!=SQLITE_OK ) return rc;
2180:
2181: rc = sqlite3_bind_int(s, 1, iLevel);
2182: if( rc!=SQLITE_OK ) return rc;
2183:
2184: rc = sqlite3_step(s);
2185: /* Should always get at least one row due to how max() works. */
2186: if( rc==SQLITE_DONE ) return SQLITE_DONE;
2187: if( rc!=SQLITE_ROW ) return rc;
2188:
2189: /* NULL means that there were no inputs to max(). */
2190: if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
2191: rc = sqlite3_step(s);
2192: if( rc==SQLITE_ROW ) return SQLITE_ERROR;
2193: return rc;
2194: }
2195:
2196: *pidx = sqlite3_column_int(s, 0);
2197:
2198: /* We expect only one row. We must execute another sqlite3_step()
2199: * to complete the iteration; otherwise the table will remain locked. */
2200: rc = sqlite3_step(s);
2201: if( rc==SQLITE_ROW ) return SQLITE_ERROR;
2202: if( rc!=SQLITE_DONE ) return rc;
2203: return SQLITE_ROW;
2204: }
2205:
2206: /* insert into %_segdir values (
2207: ** [iLevel], [idx],
2208: ** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid],
2209: ** [pRootData]
2210: ** )
2211: */
2212: static int segdir_set(fulltext_vtab *v, int iLevel, int idx,
2213: sqlite_int64 iStartBlockid,
2214: sqlite_int64 iLeavesEndBlockid,
2215: sqlite_int64 iEndBlockid,
2216: const char *pRootData, int nRootData){
2217: sqlite3_stmt *s;
2218: int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s);
2219: if( rc!=SQLITE_OK ) return rc;
2220:
2221: rc = sqlite3_bind_int(s, 1, iLevel);
2222: if( rc!=SQLITE_OK ) return rc;
2223:
2224: rc = sqlite3_bind_int(s, 2, idx);
2225: if( rc!=SQLITE_OK ) return rc;
2226:
2227: rc = sqlite3_bind_int64(s, 3, iStartBlockid);
2228: if( rc!=SQLITE_OK ) return rc;
2229:
2230: rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid);
2231: if( rc!=SQLITE_OK ) return rc;
2232:
2233: rc = sqlite3_bind_int64(s, 5, iEndBlockid);
2234: if( rc!=SQLITE_OK ) return rc;
2235:
2236: rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC);
2237: if( rc!=SQLITE_OK ) return rc;
2238:
2239: return sql_single_step(s);
2240: }
2241:
2242: /* Queries %_segdir for the block span of the segments in level
2243: ** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel,
2244: ** SQLITE_ROW if there are blocks, else an error.
2245: */
2246: static int segdir_span(fulltext_vtab *v, int iLevel,
2247: sqlite_int64 *piStartBlockid,
2248: sqlite_int64 *piEndBlockid){
2249: sqlite3_stmt *s;
2250: int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s);
2251: if( rc!=SQLITE_OK ) return rc;
2252:
2253: rc = sqlite3_bind_int(s, 1, iLevel);
2254: if( rc!=SQLITE_OK ) return rc;
2255:
2256: rc = sqlite3_step(s);
2257: if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */
2258: if( rc!=SQLITE_ROW ) return rc;
2259:
2260: /* This happens if all segments at this level are entirely inline. */
2261: if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
2262: /* We expect only one row. We must execute another sqlite3_step()
2263: * to complete the iteration; otherwise the table will remain locked. */
2264: int rc2 = sqlite3_step(s);
2265: if( rc2==SQLITE_ROW ) return SQLITE_ERROR;
2266: return rc2;
2267: }
2268:
2269: *piStartBlockid = sqlite3_column_int64(s, 0);
2270: *piEndBlockid = sqlite3_column_int64(s, 1);
2271:
2272: /* We expect only one row. We must execute another sqlite3_step()
2273: * to complete the iteration; otherwise the table will remain locked. */
2274: rc = sqlite3_step(s);
2275: if( rc==SQLITE_ROW ) return SQLITE_ERROR;
2276: if( rc!=SQLITE_DONE ) return rc;
2277: return SQLITE_ROW;
2278: }
2279:
2280: /* Delete the segment blocks and segment directory records for all
2281: ** segments at iLevel.
2282: */
2283: static int segdir_delete(fulltext_vtab *v, int iLevel){
2284: sqlite3_stmt *s;
2285: sqlite_int64 iStartBlockid, iEndBlockid;
2286: int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid);
2287: if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
2288:
2289: if( rc==SQLITE_ROW ){
2290: rc = block_delete(v, iStartBlockid, iEndBlockid);
2291: if( rc!=SQLITE_OK ) return rc;
2292: }
2293:
2294: /* Delete the segment directory itself. */
2295: rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s);
2296: if( rc!=SQLITE_OK ) return rc;
2297:
2298: rc = sqlite3_bind_int64(s, 1, iLevel);
2299: if( rc!=SQLITE_OK ) return rc;
2300:
2301: return sql_single_step(s);
2302: }
2303:
2304: /* Delete entire fts index, SQLITE_OK on success, relevant error on
2305: ** failure.
2306: */
2307: static int segdir_delete_all(fulltext_vtab *v){
2308: sqlite3_stmt *s;
2309: int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s);
2310: if( rc!=SQLITE_OK ) return rc;
2311:
2312: rc = sql_single_step(s);
2313: if( rc!=SQLITE_OK ) return rc;
2314:
2315: rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s);
2316: if( rc!=SQLITE_OK ) return rc;
2317:
2318: return sql_single_step(s);
2319: }
2320:
2321: /* Returns SQLITE_OK with *pnSegments set to the number of entries in
2322: ** %_segdir and *piMaxLevel set to the highest level which has a
2323: ** segment. Otherwise returns the SQLite error which caused failure.
2324: */
2325: static int segdir_count(fulltext_vtab *v, int *pnSegments, int *piMaxLevel){
2326: sqlite3_stmt *s;
2327: int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s);
2328: if( rc!=SQLITE_OK ) return rc;
2329:
2330: rc = sqlite3_step(s);
2331: /* TODO(shess): This case should not be possible? Should stronger
2332: ** measures be taken if it happens?
2333: */
2334: if( rc==SQLITE_DONE ){
2335: *pnSegments = 0;
2336: *piMaxLevel = 0;
2337: return SQLITE_OK;
2338: }
2339: if( rc!=SQLITE_ROW ) return rc;
2340:
2341: *pnSegments = sqlite3_column_int(s, 0);
2342: *piMaxLevel = sqlite3_column_int(s, 1);
2343:
2344: /* We expect only one row. We must execute another sqlite3_step()
2345: * to complete the iteration; otherwise the table will remain locked. */
2346: rc = sqlite3_step(s);
2347: if( rc==SQLITE_DONE ) return SQLITE_OK;
2348: if( rc==SQLITE_ROW ) return SQLITE_ERROR;
2349: return rc;
2350: }
2351:
2352: /* TODO(shess) clearPendingTerms() is far down the file because
2353: ** writeZeroSegment() is far down the file because LeafWriter is far
2354: ** down the file. Consider refactoring the code to move the non-vtab
2355: ** code above the vtab code so that we don't need this forward
2356: ** reference.
2357: */
2358: static int clearPendingTerms(fulltext_vtab *v);
2359:
2360: /*
2361: ** Free the memory used to contain a fulltext_vtab structure.
2362: */
2363: static void fulltext_vtab_destroy(fulltext_vtab *v){
2364: int iStmt, i;
2365:
2366: TRACE(("FTS2 Destroy %p\n", v));
2367: for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
2368: if( v->pFulltextStatements[iStmt]!=NULL ){
2369: sqlite3_finalize(v->pFulltextStatements[iStmt]);
2370: v->pFulltextStatements[iStmt] = NULL;
2371: }
2372: }
2373:
2374: for( i=0; i<MERGE_COUNT; i++ ){
2375: if( v->pLeafSelectStmts[i]!=NULL ){
2376: sqlite3_finalize(v->pLeafSelectStmts[i]);
2377: v->pLeafSelectStmts[i] = NULL;
2378: }
2379: }
2380:
2381: if( v->pTokenizer!=NULL ){
2382: v->pTokenizer->pModule->xDestroy(v->pTokenizer);
2383: v->pTokenizer = NULL;
2384: }
2385:
2386: clearPendingTerms(v);
2387:
2388: sqlite3_free(v->azColumn);
2389: for(i = 0; i < v->nColumn; ++i) {
2390: sqlite3_free(v->azContentColumn[i]);
2391: }
2392: sqlite3_free(v->azContentColumn);
2393: sqlite3_free(v);
2394: }
2395:
2396: /*
2397: ** Token types for parsing the arguments to xConnect or xCreate.
2398: */
2399: #define TOKEN_EOF 0 /* End of file */
2400: #define TOKEN_SPACE 1 /* Any kind of whitespace */
2401: #define TOKEN_ID 2 /* An identifier */
2402: #define TOKEN_STRING 3 /* A string literal */
2403: #define TOKEN_PUNCT 4 /* A single punctuation character */
2404:
2405: /*
2406: ** If X is a character that can be used in an identifier then
2407: ** IdChar(X) will be true. Otherwise it is false.
2408: **
2409: ** For ASCII, any character with the high-order bit set is
2410: ** allowed in an identifier. For 7-bit characters,
2411: ** sqlite3IsIdChar[X] must be 1.
2412: **
2413: ** Ticket #1066. the SQL standard does not allow '$' in the
2414: ** middle of identfiers. But many SQL implementations do.
2415: ** SQLite will allow '$' in identifiers for compatibility.
2416: ** But the feature is undocumented.
2417: */
2418: static const char isIdChar[] = {
2419: /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
2420: 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
2421: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
2422: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
2423: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
2424: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
2425: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
2426: };
2427: #define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isIdChar[c-0x20]))
2428:
2429:
2430: /*
2431: ** Return the length of the token that begins at z[0].
2432: ** Store the token type in *tokenType before returning.
2433: */
2434: static int getToken(const char *z, int *tokenType){
2435: int i, c;
2436: switch( *z ){
2437: case 0: {
2438: *tokenType = TOKEN_EOF;
2439: return 0;
2440: }
2441: case ' ': case '\t': case '\n': case '\f': case '\r': {
2442: for(i=1; safe_isspace(z[i]); i++){}
2443: *tokenType = TOKEN_SPACE;
2444: return i;
2445: }
2446: case '`':
2447: case '\'':
2448: case '"': {
2449: int delim = z[0];
2450: for(i=1; (c=z[i])!=0; i++){
2451: if( c==delim ){
2452: if( z[i+1]==delim ){
2453: i++;
2454: }else{
2455: break;
2456: }
2457: }
2458: }
2459: *tokenType = TOKEN_STRING;
2460: return i + (c!=0);
2461: }
2462: case '[': {
2463: for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
2464: *tokenType = TOKEN_ID;
2465: return i;
2466: }
2467: default: {
2468: if( !IdChar(*z) ){
2469: break;
2470: }
2471: for(i=1; IdChar(z[i]); i++){}
2472: *tokenType = TOKEN_ID;
2473: return i;
2474: }
2475: }
2476: *tokenType = TOKEN_PUNCT;
2477: return 1;
2478: }
2479:
2480: /*
2481: ** A token extracted from a string is an instance of the following
2482: ** structure.
2483: */
2484: typedef struct Token {
2485: const char *z; /* Pointer to token text. Not '\000' terminated */
2486: short int n; /* Length of the token text in bytes. */
2487: } Token;
2488:
2489: /*
2490: ** Given a input string (which is really one of the argv[] parameters
2491: ** passed into xConnect or xCreate) split the string up into tokens.
2492: ** Return an array of pointers to '\000' terminated strings, one string
2493: ** for each non-whitespace token.
2494: **
2495: ** The returned array is terminated by a single NULL pointer.
2496: **
2497: ** Space to hold the returned array is obtained from a single
2498: ** malloc and should be freed by passing the return value to free().
2499: ** The individual strings within the token list are all a part of
2500: ** the single memory allocation and will all be freed at once.
2501: */
2502: static char **tokenizeString(const char *z, int *pnToken){
2503: int nToken = 0;
2504: Token *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) );
2505: int n = 1;
2506: int e, i;
2507: int totalSize = 0;
2508: char **azToken;
2509: char *zCopy;
2510: while( n>0 ){
2511: n = getToken(z, &e);
2512: if( e!=TOKEN_SPACE ){
2513: aToken[nToken].z = z;
2514: aToken[nToken].n = n;
2515: nToken++;
2516: totalSize += n+1;
2517: }
2518: z += n;
2519: }
2520: azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize );
2521: zCopy = (char*)&azToken[nToken];
2522: nToken--;
2523: for(i=0; i<nToken; i++){
2524: azToken[i] = zCopy;
2525: n = aToken[i].n;
2526: memcpy(zCopy, aToken[i].z, n);
2527: zCopy[n] = 0;
2528: zCopy += n+1;
2529: }
2530: azToken[nToken] = 0;
2531: sqlite3_free(aToken);
2532: *pnToken = nToken;
2533: return azToken;
2534: }
2535:
2536: /*
2537: ** Convert an SQL-style quoted string into a normal string by removing
2538: ** the quote characters. The conversion is done in-place. If the
2539: ** input does not begin with a quote character, then this routine
2540: ** is a no-op.
2541: **
2542: ** Examples:
2543: **
2544: ** "abc" becomes abc
2545: ** 'xyz' becomes xyz
2546: ** [pqr] becomes pqr
2547: ** `mno` becomes mno
2548: */
2549: static void dequoteString(char *z){
2550: int quote;
2551: int i, j;
2552: if( z==0 ) return;
2553: quote = z[0];
2554: switch( quote ){
2555: case '\'': break;
2556: case '"': break;
2557: case '`': break; /* For MySQL compatibility */
2558: case '[': quote = ']'; break; /* For MS SqlServer compatibility */
2559: default: return;
2560: }
2561: for(i=1, j=0; z[i]; i++){
2562: if( z[i]==quote ){
2563: if( z[i+1]==quote ){
2564: z[j++] = quote;
2565: i++;
2566: }else{
2567: z[j++] = 0;
2568: break;
2569: }
2570: }else{
2571: z[j++] = z[i];
2572: }
2573: }
2574: }
2575:
2576: /*
2577: ** The input azIn is a NULL-terminated list of tokens. Remove the first
2578: ** token and all punctuation tokens. Remove the quotes from
2579: ** around string literal tokens.
2580: **
2581: ** Example:
2582: **
2583: ** input: tokenize chinese ( 'simplifed' , 'mixed' )
2584: ** output: chinese simplifed mixed
2585: **
2586: ** Another example:
2587: **
2588: ** input: delimiters ( '[' , ']' , '...' )
2589: ** output: [ ] ...
2590: */
2591: static void tokenListToIdList(char **azIn){
2592: int i, j;
2593: if( azIn ){
2594: for(i=0, j=-1; azIn[i]; i++){
2595: if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){
2596: dequoteString(azIn[i]);
2597: if( j>=0 ){
2598: azIn[j] = azIn[i];
2599: }
2600: j++;
2601: }
2602: }
2603: azIn[j] = 0;
2604: }
2605: }
2606:
2607:
2608: /*
2609: ** Find the first alphanumeric token in the string zIn. Null-terminate
2610: ** this token. Remove any quotation marks. And return a pointer to
2611: ** the result.
2612: */
2613: static char *firstToken(char *zIn, char **pzTail){
2614: int n, ttype;
2615: while(1){
2616: n = getToken(zIn, &ttype);
2617: if( ttype==TOKEN_SPACE ){
2618: zIn += n;
2619: }else if( ttype==TOKEN_EOF ){
2620: *pzTail = zIn;
2621: return 0;
2622: }else{
2623: zIn[n] = 0;
2624: *pzTail = &zIn[1];
2625: dequoteString(zIn);
2626: return zIn;
2627: }
2628: }
2629: /*NOTREACHED*/
2630: }
2631:
2632: /* Return true if...
2633: **
2634: ** * s begins with the string t, ignoring case
2635: ** * s is longer than t
2636: ** * The first character of s beyond t is not a alphanumeric
2637: **
2638: ** Ignore leading space in *s.
2639: **
2640: ** To put it another way, return true if the first token of
2641: ** s[] is t[].
2642: */
2643: static int startsWith(const char *s, const char *t){
2644: while( safe_isspace(*s) ){ s++; }
2645: while( *t ){
2646: if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0;
2647: }
2648: return *s!='_' && !safe_isalnum(*s);
2649: }
2650:
2651: /*
2652: ** An instance of this structure defines the "spec" of a
2653: ** full text index. This structure is populated by parseSpec
2654: ** and use by fulltextConnect and fulltextCreate.
2655: */
2656: typedef struct TableSpec {
2657: const char *zDb; /* Logical database name */
2658: const char *zName; /* Name of the full-text index */
2659: int nColumn; /* Number of columns to be indexed */
2660: char **azColumn; /* Original names of columns to be indexed */
2661: char **azContentColumn; /* Column names for %_content */
2662: char **azTokenizer; /* Name of tokenizer and its arguments */
2663: } TableSpec;
2664:
2665: /*
2666: ** Reclaim all of the memory used by a TableSpec
2667: */
2668: static void clearTableSpec(TableSpec *p) {
2669: sqlite3_free(p->azColumn);
2670: sqlite3_free(p->azContentColumn);
2671: sqlite3_free(p->azTokenizer);
2672: }
2673:
2674: /* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
2675: *
2676: * CREATE VIRTUAL TABLE email
2677: * USING fts2(subject, body, tokenize mytokenizer(myarg))
2678: *
2679: * We return parsed information in a TableSpec structure.
2680: *
2681: */
2682: static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv,
2683: char**pzErr){
2684: int i, n;
2685: char *z, *zDummy;
2686: char **azArg;
2687: const char *zTokenizer = 0; /* argv[] entry describing the tokenizer */
2688:
2689: assert( argc>=3 );
2690: /* Current interface:
2691: ** argv[0] - module name
2692: ** argv[1] - database name
2693: ** argv[2] - table name
2694: ** argv[3..] - columns, optionally followed by tokenizer specification
2695: ** and snippet delimiters specification.
2696: */
2697:
2698: /* Make a copy of the complete argv[][] array in a single allocation.
2699: ** The argv[][] array is read-only and transient. We can write to the
2700: ** copy in order to modify things and the copy is persistent.
2701: */
2702: CLEAR(pSpec);
2703: for(i=n=0; i<argc; i++){
2704: n += strlen(argv[i]) + 1;
2705: }
2706: azArg = sqlite3_malloc( sizeof(char*)*argc + n );
2707: if( azArg==0 ){
2708: return SQLITE_NOMEM;
2709: }
2710: z = (char*)&azArg[argc];
2711: for(i=0; i<argc; i++){
2712: azArg[i] = z;
2713: strcpy(z, argv[i]);
2714: z += strlen(z)+1;
2715: }
2716:
2717: /* Identify the column names and the tokenizer and delimiter arguments
2718: ** in the argv[][] array.
2719: */
2720: pSpec->zDb = azArg[1];
2721: pSpec->zName = azArg[2];
2722: pSpec->nColumn = 0;
2723: pSpec->azColumn = azArg;
2724: zTokenizer = "tokenize simple";
2725: for(i=3; i<argc; ++i){
2726: if( startsWith(azArg[i],"tokenize") ){
2727: zTokenizer = azArg[i];
2728: }else{
2729: z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
2730: pSpec->nColumn++;
2731: }
2732: }
2733: if( pSpec->nColumn==0 ){
2734: azArg[0] = "content";
2735: pSpec->nColumn = 1;
2736: }
2737:
2738: /*
2739: ** Construct the list of content column names.
2740: **
2741: ** Each content column name will be of the form cNNAAAA
2742: ** where NN is the column number and AAAA is the sanitized
2743: ** column name. "sanitized" means that special characters are
2744: ** converted to "_". The cNN prefix guarantees that all column
2745: ** names are unique.
2746: **
2747: ** The AAAA suffix is not strictly necessary. It is included
2748: ** for the convenience of people who might examine the generated
2749: ** %_content table and wonder what the columns are used for.
2750: */
2751: pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) );
2752: if( pSpec->azContentColumn==0 ){
2753: clearTableSpec(pSpec);
2754: return SQLITE_NOMEM;
2755: }
2756: for(i=0; i<pSpec->nColumn; i++){
2757: char *p;
2758: pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
2759: for (p = pSpec->azContentColumn[i]; *p ; ++p) {
2760: if( !safe_isalnum(*p) ) *p = '_';
2761: }
2762: }
2763:
2764: /*
2765: ** Parse the tokenizer specification string.
2766: */
2767: pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
2768: tokenListToIdList(pSpec->azTokenizer);
2769:
2770: return SQLITE_OK;
2771: }
2772:
2773: /*
2774: ** Generate a CREATE TABLE statement that describes the schema of
2775: ** the virtual table. Return a pointer to this schema string.
2776: **
2777: ** Space is obtained from sqlite3_mprintf() and should be freed
2778: ** using sqlite3_free().
2779: */
2780: static char *fulltextSchema(
2781: int nColumn, /* Number of columns */
2782: const char *const* azColumn, /* List of columns */
2783: const char *zTableName /* Name of the table */
2784: ){
2785: int i;
2786: char *zSchema, *zNext;
2787: const char *zSep = "(";
2788: zSchema = sqlite3_mprintf("CREATE TABLE x");
2789: for(i=0; i<nColumn; i++){
2790: zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
2791: sqlite3_free(zSchema);
2792: zSchema = zNext;
2793: zSep = ",";
2794: }
2795: zNext = sqlite3_mprintf("%s,%Q)", zSchema, zTableName);
2796: sqlite3_free(zSchema);
2797: return zNext;
2798: }
2799:
2800: /*
2801: ** Build a new sqlite3_vtab structure that will describe the
2802: ** fulltext index defined by spec.
2803: */
2804: static int constructVtab(
2805: sqlite3 *db, /* The SQLite database connection */
2806: fts2Hash *pHash, /* Hash table containing tokenizers */
2807: TableSpec *spec, /* Parsed spec information from parseSpec() */
2808: sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
2809: char **pzErr /* Write any error message here */
2810: ){
2811: int rc;
2812: int n;
2813: fulltext_vtab *v = 0;
2814: const sqlite3_tokenizer_module *m = NULL;
2815: char *schema;
2816:
2817: char const *zTok; /* Name of tokenizer to use for this fts table */
2818: int nTok; /* Length of zTok, including nul terminator */
2819:
2820: v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab));
2821: if( v==0 ) return SQLITE_NOMEM;
2822: CLEAR(v);
2823: /* sqlite will initialize v->base */
2824: v->db = db;
2825: v->zDb = spec->zDb; /* Freed when azColumn is freed */
2826: v->zName = spec->zName; /* Freed when azColumn is freed */
2827: v->nColumn = spec->nColumn;
2828: v->azContentColumn = spec->azContentColumn;
2829: spec->azContentColumn = 0;
2830: v->azColumn = spec->azColumn;
2831: spec->azColumn = 0;
2832:
2833: if( spec->azTokenizer==0 ){
2834: return SQLITE_NOMEM;
2835: }
2836:
2837: zTok = spec->azTokenizer[0];
2838: if( !zTok ){
2839: zTok = "simple";
2840: }
2841: nTok = strlen(zTok)+1;
2842:
2843: m = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zTok, nTok);
2844: if( !m ){
2845: *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
2846: rc = SQLITE_ERROR;
2847: goto err;
2848: }
2849:
2850: for(n=0; spec->azTokenizer[n]; n++){}
2851: if( n ){
2852: rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1],
2853: &v->pTokenizer);
2854: }else{
2855: rc = m->xCreate(0, 0, &v->pTokenizer);
2856: }
2857: if( rc!=SQLITE_OK ) goto err;
2858: v->pTokenizer->pModule = m;
2859:
2860: /* TODO: verify the existence of backing tables foo_content, foo_term */
2861:
2862: schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn,
2863: spec->zName);
2864: rc = sqlite3_declare_vtab(db, schema);
2865: sqlite3_free(schema);
2866: if( rc!=SQLITE_OK ) goto err;
2867:
2868: memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
2869:
2870: /* Indicate that the buffer is not live. */
2871: v->nPendingData = -1;
2872:
2873: *ppVTab = &v->base;
2874: TRACE(("FTS2 Connect %p\n", v));
2875:
2876: return rc;
2877:
2878: err:
2879: fulltext_vtab_destroy(v);
2880: return rc;
2881: }
2882:
2883: static int fulltextConnect(
2884: sqlite3 *db,
2885: void *pAux,
2886: int argc, const char *const*argv,
2887: sqlite3_vtab **ppVTab,
2888: char **pzErr
2889: ){
2890: TableSpec spec;
2891: int rc = parseSpec(&spec, argc, argv, pzErr);
2892: if( rc!=SQLITE_OK ) return rc;
2893:
2894: rc = constructVtab(db, (fts2Hash *)pAux, &spec, ppVTab, pzErr);
2895: clearTableSpec(&spec);
2896: return rc;
2897: }
2898:
2899: /* The %_content table holds the text of each document, with
2900: ** the rowid used as the docid.
2901: */
2902: /* TODO(shess) This comment needs elaboration to match the updated
2903: ** code. Work it into the top-of-file comment at that time.
2904: */
2905: static int fulltextCreate(sqlite3 *db, void *pAux,
2906: int argc, const char * const *argv,
2907: sqlite3_vtab **ppVTab, char **pzErr){
2908: int rc;
2909: TableSpec spec;
2910: StringBuffer schema;
2911: TRACE(("FTS2 Create\n"));
2912:
2913: rc = parseSpec(&spec, argc, argv, pzErr);
2914: if( rc!=SQLITE_OK ) return rc;
2915:
2916: initStringBuffer(&schema);
2917: append(&schema, "CREATE TABLE %_content(");
2918: appendList(&schema, spec.nColumn, spec.azContentColumn);
2919: append(&schema, ")");
2920: rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema));
2921: stringBufferDestroy(&schema);
2922: if( rc!=SQLITE_OK ) goto out;
2923:
2924: rc = sql_exec(db, spec.zDb, spec.zName,
2925: "create table %_segments(block blob);");
2926: if( rc!=SQLITE_OK ) goto out;
2927:
2928: rc = sql_exec(db, spec.zDb, spec.zName,
2929: "create table %_segdir("
2930: " level integer,"
2931: " idx integer,"
2932: " start_block integer,"
2933: " leaves_end_block integer,"
2934: " end_block integer,"
2935: " root blob,"
2936: " primary key(level, idx)"
2937: ");");
2938: if( rc!=SQLITE_OK ) goto out;
2939:
2940: rc = constructVtab(db, (fts2Hash *)pAux, &spec, ppVTab, pzErr);
2941:
2942: out:
2943: clearTableSpec(&spec);
2944: return rc;
2945: }
2946:
2947: /* Decide how to handle an SQL query. */
2948: static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
2949: int i;
2950: TRACE(("FTS2 BestIndex\n"));
2951:
2952: for(i=0; i<pInfo->nConstraint; ++i){
2953: const struct sqlite3_index_constraint *pConstraint;
2954: pConstraint = &pInfo->aConstraint[i];
2955: if( pConstraint->usable ) {
2956: if( pConstraint->iColumn==-1 &&
2957: pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
2958: pInfo->idxNum = QUERY_ROWID; /* lookup by rowid */
2959: TRACE(("FTS2 QUERY_ROWID\n"));
2960: } else if( pConstraint->iColumn>=0 &&
2961: pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
2962: /* full-text search */
2963: pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
2964: TRACE(("FTS2 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
2965: } else continue;
2966:
2967: pInfo->aConstraintUsage[i].argvIndex = 1;
2968: pInfo->aConstraintUsage[i].omit = 1;
2969:
2970: /* An arbitrary value for now.
2971: * TODO: Perhaps rowid matches should be considered cheaper than
2972: * full-text searches. */
2973: pInfo->estimatedCost = 1.0;
2974:
2975: return SQLITE_OK;
2976: }
2977: }
2978: pInfo->idxNum = QUERY_GENERIC;
2979: return SQLITE_OK;
2980: }
2981:
2982: static int fulltextDisconnect(sqlite3_vtab *pVTab){
2983: TRACE(("FTS2 Disconnect %p\n", pVTab));
2984: fulltext_vtab_destroy((fulltext_vtab *)pVTab);
2985: return SQLITE_OK;
2986: }
2987:
2988: static int fulltextDestroy(sqlite3_vtab *pVTab){
2989: fulltext_vtab *v = (fulltext_vtab *)pVTab;
2990: int rc;
2991:
2992: TRACE(("FTS2 Destroy %p\n", pVTab));
2993: rc = sql_exec(v->db, v->zDb, v->zName,
2994: "drop table if exists %_content;"
2995: "drop table if exists %_segments;"
2996: "drop table if exists %_segdir;"
2997: );
2998: if( rc!=SQLITE_OK ) return rc;
2999:
3000: fulltext_vtab_destroy((fulltext_vtab *)pVTab);
3001: return SQLITE_OK;
3002: }
3003:
3004: static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
3005: fulltext_cursor *c;
3006:
3007: c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor));
3008: if( c ){
3009: memset(c, 0, sizeof(fulltext_cursor));
3010: /* sqlite will initialize c->base */
3011: *ppCursor = &c->base;
3012: TRACE(("FTS2 Open %p: %p\n", pVTab, c));
3013: return SQLITE_OK;
3014: }else{
3015: return SQLITE_NOMEM;
3016: }
3017: }
3018:
3019:
3020: /* Free all of the dynamically allocated memory held by *q
3021: */
3022: static void queryClear(Query *q){
3023: int i;
3024: for(i = 0; i < q->nTerms; ++i){
3025: sqlite3_free(q->pTerms[i].pTerm);
3026: }
3027: sqlite3_free(q->pTerms);
3028: CLEAR(q);
3029: }
3030:
3031: /* Free all of the dynamically allocated memory held by the
3032: ** Snippet
3033: */
3034: static void snippetClear(Snippet *p){
3035: sqlite3_free(p->aMatch);
3036: sqlite3_free(p->zOffset);
3037: sqlite3_free(p->zSnippet);
3038: CLEAR(p);
3039: }
3040: /*
3041: ** Append a single entry to the p->aMatch[] log.
3042: */
3043: static void snippetAppendMatch(
3044: Snippet *p, /* Append the entry to this snippet */
3045: int iCol, int iTerm, /* The column and query term */
3046: int iStart, int nByte /* Offset and size of the match */
3047: ){
3048: int i;
3049: struct snippetMatch *pMatch;
3050: if( p->nMatch+1>=p->nAlloc ){
3051: p->nAlloc = p->nAlloc*2 + 10;
3052: p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
3053: if( p->aMatch==0 ){
3054: p->nMatch = 0;
3055: p->nAlloc = 0;
3056: return;
3057: }
3058: }
3059: i = p->nMatch++;
3060: pMatch = &p->aMatch[i];
3061: pMatch->iCol = iCol;
3062: pMatch->iTerm = iTerm;
3063: pMatch->iStart = iStart;
3064: pMatch->nByte = nByte;
3065: }
3066:
3067: /*
3068: ** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
3069: */
3070: #define FTS2_ROTOR_SZ (32)
3071: #define FTS2_ROTOR_MASK (FTS2_ROTOR_SZ-1)
3072:
3073: /*
3074: ** Add entries to pSnippet->aMatch[] for every match that occurs against
3075: ** document zDoc[0..nDoc-1] which is stored in column iColumn.
3076: */
3077: static void snippetOffsetsOfColumn(
3078: Query *pQuery,
3079: Snippet *pSnippet,
3080: int iColumn,
3081: const char *zDoc,
3082: int nDoc
3083: ){
3084: const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */
3085: sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */
3086: sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */
3087: fulltext_vtab *pVtab; /* The full text index */
3088: int nColumn; /* Number of columns in the index */
3089: const QueryTerm *aTerm; /* Query string terms */
3090: int nTerm; /* Number of query string terms */
3091: int i, j; /* Loop counters */
3092: int rc; /* Return code */
3093: unsigned int match, prevMatch; /* Phrase search bitmasks */
3094: const char *zToken; /* Next token from the tokenizer */
3095: int nToken; /* Size of zToken */
3096: int iBegin, iEnd, iPos; /* Offsets of beginning and end */
3097:
3098: /* The following variables keep a circular buffer of the last
3099: ** few tokens */
3100: unsigned int iRotor = 0; /* Index of current token */
3101: int iRotorBegin[FTS2_ROTOR_SZ]; /* Beginning offset of token */
3102: int iRotorLen[FTS2_ROTOR_SZ]; /* Length of token */
3103:
3104: pVtab = pQuery->pFts;
3105: nColumn = pVtab->nColumn;
3106: pTokenizer = pVtab->pTokenizer;
3107: pTModule = pTokenizer->pModule;
3108: rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
3109: if( rc ) return;
3110: pTCursor->pTokenizer = pTokenizer;
3111: aTerm = pQuery->pTerms;
3112: nTerm = pQuery->nTerms;
3113: if( nTerm>=FTS2_ROTOR_SZ ){
3114: nTerm = FTS2_ROTOR_SZ - 1;
3115: }
3116: prevMatch = 0;
3117: while(1){
3118: rc = pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
3119: if( rc ) break;
3120: iRotorBegin[iRotor&FTS2_ROTOR_MASK] = iBegin;
3121: iRotorLen[iRotor&FTS2_ROTOR_MASK] = iEnd-iBegin;
3122: match = 0;
3123: for(i=0; i<nTerm; i++){
3124: int iCol;
3125: iCol = aTerm[i].iColumn;
3126: if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
3127: if( aTerm[i].nTerm>nToken ) continue;
3128: if( !aTerm[i].isPrefix && aTerm[i].nTerm<nToken ) continue;
3129: assert( aTerm[i].nTerm<=nToken );
3130: if( memcmp(aTerm[i].pTerm, zToken, aTerm[i].nTerm) ) continue;
3131: if( aTerm[i].iPhrase>1 && (prevMatch & (1<<i))==0 ) continue;
3132: match |= 1<<i;
3133: if( i==nTerm-1 || aTerm[i+1].iPhrase==1 ){
3134: for(j=aTerm[i].iPhrase-1; j>=0; j--){
3135: int k = (iRotor-j) & FTS2_ROTOR_MASK;
3136: snippetAppendMatch(pSnippet, iColumn, i-j,
3137: iRotorBegin[k], iRotorLen[k]);
3138: }
3139: }
3140: }
3141: prevMatch = match<<1;
3142: iRotor++;
3143: }
3144: pTModule->xClose(pTCursor);
3145: }
3146:
3147:
3148: /*
3149: ** Compute all offsets for the current row of the query.
3150: ** If the offsets have already been computed, this routine is a no-op.
3151: */
3152: static void snippetAllOffsets(fulltext_cursor *p){
3153: int nColumn;
3154: int iColumn, i;
3155: int iFirst, iLast;
3156: fulltext_vtab *pFts;
3157:
3158: if( p->snippet.nMatch ) return;
3159: if( p->q.nTerms==0 ) return;
3160: pFts = p->q.pFts;
3161: nColumn = pFts->nColumn;
3162: iColumn = (p->iCursorType - QUERY_FULLTEXT);
3163: if( iColumn<0 || iColumn>=nColumn ){
3164: iFirst = 0;
3165: iLast = nColumn-1;
3166: }else{
3167: iFirst = iColumn;
3168: iLast = iColumn;
3169: }
3170: for(i=iFirst; i<=iLast; i++){
3171: const char *zDoc;
3172: int nDoc;
3173: zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
3174: nDoc = sqlite3_column_bytes(p->pStmt, i+1);
3175: snippetOffsetsOfColumn(&p->q, &p->snippet, i, zDoc, nDoc);
3176: }
3177: }
3178:
3179: /*
3180: ** Convert the information in the aMatch[] array of the snippet
3181: ** into the string zOffset[0..nOffset-1].
3182: */
3183: static void snippetOffsetText(Snippet *p){
3184: int i;
3185: int cnt = 0;
3186: StringBuffer sb;
3187: char zBuf[200];
3188: if( p->zOffset ) return;
3189: initStringBuffer(&sb);
3190: for(i=0; i<p->nMatch; i++){
3191: struct snippetMatch *pMatch = &p->aMatch[i];
3192: zBuf[0] = ' ';
3193: sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
3194: pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
3195: append(&sb, zBuf);
3196: cnt++;
3197: }
3198: p->zOffset = stringBufferData(&sb);
3199: p->nOffset = stringBufferLength(&sb);
3200: }
3201:
3202: /*
3203: ** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
3204: ** of matching words some of which might be in zDoc. zDoc is column
3205: ** number iCol.
3206: **
3207: ** iBreak is suggested spot in zDoc where we could begin or end an
3208: ** excerpt. Return a value similar to iBreak but possibly adjusted
3209: ** to be a little left or right so that the break point is better.
3210: */
3211: static int wordBoundary(
3212: int iBreak, /* The suggested break point */
3213: const char *zDoc, /* Document text */
3214: int nDoc, /* Number of bytes in zDoc[] */
3215: struct snippetMatch *aMatch, /* Matching words */
3216: int nMatch, /* Number of entries in aMatch[] */
3217: int iCol /* The column number for zDoc[] */
3218: ){
3219: int i;
3220: if( iBreak<=10 ){
3221: return 0;
3222: }
3223: if( iBreak>=nDoc-10 ){
3224: return nDoc;
3225: }
3226: for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
3227: while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
3228: if( i<nMatch ){
3229: if( aMatch[i].iStart<iBreak+10 ){
3230: return aMatch[i].iStart;
3231: }
3232: if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
3233: return aMatch[i-1].iStart;
3234: }
3235: }
3236: for(i=1; i<=10; i++){
3237: if( safe_isspace(zDoc[iBreak-i]) ){
3238: return iBreak - i + 1;
3239: }
3240: if( safe_isspace(zDoc[iBreak+i]) ){
3241: return iBreak + i + 1;
3242: }
3243: }
3244: return iBreak;
3245: }
3246:
3247:
3248:
3249: /*
3250: ** Allowed values for Snippet.aMatch[].snStatus
3251: */
3252: #define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
3253: #define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
3254:
3255: /*
3256: ** Generate the text of a snippet.
3257: */
3258: static void snippetText(
3259: fulltext_cursor *pCursor, /* The cursor we need the snippet for */
3260: const char *zStartMark, /* Markup to appear before each match */
3261: const char *zEndMark, /* Markup to appear after each match */
3262: const char *zEllipsis /* Ellipsis mark */
3263: ){
3264: int i, j;
3265: struct snippetMatch *aMatch;
3266: int nMatch;
3267: int nDesired;
3268: StringBuffer sb;
3269: int tailCol;
3270: int tailOffset;
3271: int iCol;
3272: int nDoc;
3273: const char *zDoc;
3274: int iStart, iEnd;
3275: int tailEllipsis = 0;
3276: int iMatch;
3277:
3278:
3279: sqlite3_free(pCursor->snippet.zSnippet);
3280: pCursor->snippet.zSnippet = 0;
3281: aMatch = pCursor->snippet.aMatch;
3282: nMatch = pCursor->snippet.nMatch;
3283: initStringBuffer(&sb);
3284:
3285: for(i=0; i<nMatch; i++){
3286: aMatch[i].snStatus = SNIPPET_IGNORE;
3287: }
3288: nDesired = 0;
3289: for(i=0; i<pCursor->q.nTerms; i++){
3290: for(j=0; j<nMatch; j++){
3291: if( aMatch[j].iTerm==i ){
3292: aMatch[j].snStatus = SNIPPET_DESIRED;
3293: nDesired++;
3294: break;
3295: }
3296: }
3297: }
3298:
3299: iMatch = 0;
3300: tailCol = -1;
3301: tailOffset = 0;
3302: for(i=0; i<nMatch && nDesired>0; i++){
3303: if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
3304: nDesired--;
3305: iCol = aMatch[i].iCol;
3306: zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
3307: nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
3308: iStart = aMatch[i].iStart - 40;
3309: iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
3310: if( iStart<=10 ){
3311: iStart = 0;
3312: }
3313: if( iCol==tailCol && iStart<=tailOffset+20 ){
3314: iStart = tailOffset;
3315: }
3316: if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
3317: trimWhiteSpace(&sb);
3318: appendWhiteSpace(&sb);
3319: append(&sb, zEllipsis);
3320: appendWhiteSpace(&sb);
3321: }
3322: iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
3323: iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
3324: if( iEnd>=nDoc-10 ){
3325: iEnd = nDoc;
3326: tailEllipsis = 0;
3327: }else{
3328: tailEllipsis = 1;
3329: }
3330: while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
3331: while( iStart<iEnd ){
3332: while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
3333: && aMatch[iMatch].iCol<=iCol ){
3334: iMatch++;
3335: }
3336: if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
3337: && aMatch[iMatch].iCol==iCol ){
3338: nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
3339: iStart = aMatch[iMatch].iStart;
3340: append(&sb, zStartMark);
3341: nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
3342: append(&sb, zEndMark);
3343: iStart += aMatch[iMatch].nByte;
3344: for(j=iMatch+1; j<nMatch; j++){
3345: if( aMatch[j].iTerm==aMatch[iMatch].iTerm
3346: && aMatch[j].snStatus==SNIPPET_DESIRED ){
3347: nDesired--;
3348: aMatch[j].snStatus = SNIPPET_IGNORE;
3349: }
3350: }
3351: }else{
3352: nappend(&sb, &zDoc[iStart], iEnd - iStart);
3353: iStart = iEnd;
3354: }
3355: }
3356: tailCol = iCol;
3357: tailOffset = iEnd;
3358: }
3359: trimWhiteSpace(&sb);
3360: if( tailEllipsis ){
3361: appendWhiteSpace(&sb);
3362: append(&sb, zEllipsis);
3363: }
3364: pCursor->snippet.zSnippet = stringBufferData(&sb);
3365: pCursor->snippet.nSnippet = stringBufferLength(&sb);
3366: }
3367:
3368:
3369: /*
3370: ** Close the cursor. For additional information see the documentation
3371: ** on the xClose method of the virtual table interface.
3372: */
3373: static int fulltextClose(sqlite3_vtab_cursor *pCursor){
3374: fulltext_cursor *c = (fulltext_cursor *) pCursor;
3375: TRACE(("FTS2 Close %p\n", c));
3376: sqlite3_finalize(c->pStmt);
3377: queryClear(&c->q);
3378: snippetClear(&c->snippet);
3379: if( c->result.nData!=0 ) dlrDestroy(&c->reader);
3380: dataBufferDestroy(&c->result);
3381: sqlite3_free(c);
3382: return SQLITE_OK;
3383: }
3384:
3385: static int fulltextNext(sqlite3_vtab_cursor *pCursor){
3386: fulltext_cursor *c = (fulltext_cursor *) pCursor;
3387: int rc;
3388:
3389: TRACE(("FTS2 Next %p\n", pCursor));
3390: snippetClear(&c->snippet);
3391: if( c->iCursorType < QUERY_FULLTEXT ){
3392: /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
3393: rc = sqlite3_step(c->pStmt);
3394: switch( rc ){
3395: case SQLITE_ROW:
3396: c->eof = 0;
3397: return SQLITE_OK;
3398: case SQLITE_DONE:
3399: c->eof = 1;
3400: return SQLITE_OK;
3401: default:
3402: c->eof = 1;
3403: return rc;
3404: }
3405: } else { /* full-text query */
3406: rc = sqlite3_reset(c->pStmt);
3407: if( rc!=SQLITE_OK ) return rc;
3408:
3409: if( c->result.nData==0 || dlrAtEnd(&c->reader) ){
3410: c->eof = 1;
3411: return SQLITE_OK;
3412: }
3413: rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
3414: dlrStep(&c->reader);
3415: if( rc!=SQLITE_OK ) return rc;
3416: /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
3417: rc = sqlite3_step(c->pStmt);
3418: if( rc==SQLITE_ROW ){ /* the case we expect */
3419: c->eof = 0;
3420: return SQLITE_OK;
3421: }
3422: /* an error occurred; abort */
3423: return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
3424: }
3425: }
3426:
3427:
3428: /* TODO(shess) If we pushed LeafReader to the top of the file, or to
3429: ** another file, term_select() could be pushed above
3430: ** docListOfTerm().
3431: */
3432: static int termSelect(fulltext_vtab *v, int iColumn,
3433: const char *pTerm, int nTerm, int isPrefix,
3434: DocListType iType, DataBuffer *out);
3435:
3436: /* Return a DocList corresponding to the query term *pTerm. If *pTerm
3437: ** is the first term of a phrase query, go ahead and evaluate the phrase
3438: ** query and return the doclist for the entire phrase query.
3439: **
3440: ** The resulting DL_DOCIDS doclist is stored in pResult, which is
3441: ** overwritten.
3442: */
3443: static int docListOfTerm(
3444: fulltext_vtab *v, /* The full text index */
3445: int iColumn, /* column to restrict to. No restriction if >=nColumn */
3446: QueryTerm *pQTerm, /* Term we are looking for, or 1st term of a phrase */
3447: DataBuffer *pResult /* Write the result here */
3448: ){
3449: DataBuffer left, right, new;
3450: int i, rc;
3451:
3452: /* No phrase search if no position info. */
3453: assert( pQTerm->nPhrase==0 || DL_DEFAULT!=DL_DOCIDS );
3454:
3455: /* This code should never be called with buffered updates. */
3456: assert( v->nPendingData<0 );
3457:
3458: dataBufferInit(&left, 0);
3459: rc = termSelect(v, iColumn, pQTerm->pTerm, pQTerm->nTerm, pQTerm->isPrefix,
3460: 0<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &left);
3461: if( rc ) return rc;
3462: for(i=1; i<=pQTerm->nPhrase && left.nData>0; i++){
3463: dataBufferInit(&right, 0);
3464: rc = termSelect(v, iColumn, pQTerm[i].pTerm, pQTerm[i].nTerm,
3465: pQTerm[i].isPrefix, DL_POSITIONS, &right);
3466: if( rc ){
3467: dataBufferDestroy(&left);
3468: return rc;
3469: }
3470: dataBufferInit(&new, 0);
3471: docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
3472: i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &new);
3473: dataBufferDestroy(&left);
3474: dataBufferDestroy(&right);
3475: left = new;
3476: }
3477: *pResult = left;
3478: return SQLITE_OK;
3479: }
3480:
3481: /* Add a new term pTerm[0..nTerm-1] to the query *q.
3482: */
3483: static void queryAdd(Query *q, const char *pTerm, int nTerm){
3484: QueryTerm *t;
3485: ++q->nTerms;
3486: q->pTerms = sqlite3_realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0]));
3487: if( q->pTerms==0 ){
3488: q->nTerms = 0;
3489: return;
3490: }
3491: t = &q->pTerms[q->nTerms - 1];
3492: CLEAR(t);
3493: t->pTerm = sqlite3_malloc(nTerm+1);
3494: memcpy(t->pTerm, pTerm, nTerm);
3495: t->pTerm[nTerm] = 0;
3496: t->nTerm = nTerm;
3497: t->isOr = q->nextIsOr;
3498: t->isPrefix = 0;
3499: q->nextIsOr = 0;
3500: t->iColumn = q->nextColumn;
3501: q->nextColumn = q->dfltColumn;
3502: }
3503:
3504: /*
3505: ** Check to see if the string zToken[0...nToken-1] matches any
3506: ** column name in the virtual table. If it does,
3507: ** return the zero-indexed column number. If not, return -1.
3508: */
3509: static int checkColumnSpecifier(
3510: fulltext_vtab *pVtab, /* The virtual table */
3511: const char *zToken, /* Text of the token */
3512: int nToken /* Number of characters in the token */
3513: ){
3514: int i;
3515: for(i=0; i<pVtab->nColumn; i++){
3516: if( memcmp(pVtab->azColumn[i], zToken, nToken)==0
3517: && pVtab->azColumn[i][nToken]==0 ){
3518: return i;
3519: }
3520: }
3521: return -1;
3522: }
3523:
3524: /*
3525: ** Parse the text at pSegment[0..nSegment-1]. Add additional terms
3526: ** to the query being assemblied in pQuery.
3527: **
3528: ** inPhrase is true if pSegment[0..nSegement-1] is contained within
3529: ** double-quotes. If inPhrase is true, then the first term
3530: ** is marked with the number of terms in the phrase less one and
3531: ** OR and "-" syntax is ignored. If inPhrase is false, then every
3532: ** term found is marked with nPhrase=0 and OR and "-" syntax is significant.
3533: */
3534: static int tokenizeSegment(
3535: sqlite3_tokenizer *pTokenizer, /* The tokenizer to use */
3536: const char *pSegment, int nSegment, /* Query expression being parsed */
3537: int inPhrase, /* True if within "..." */
3538: Query *pQuery /* Append results here */
3539: ){
3540: const sqlite3_tokenizer_module *pModule = pTokenizer->pModule;
3541: sqlite3_tokenizer_cursor *pCursor;
3542: int firstIndex = pQuery->nTerms;
3543: int iCol;
3544: int nTerm = 1;
3545:
3546: int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor);
3547: if( rc!=SQLITE_OK ) return rc;
3548: pCursor->pTokenizer = pTokenizer;
3549:
3550: while( 1 ){
3551: const char *pToken;
3552: int nToken, iBegin, iEnd, iPos;
3553:
3554: rc = pModule->xNext(pCursor,
3555: &pToken, &nToken,
3556: &iBegin, &iEnd, &iPos);
3557: if( rc!=SQLITE_OK ) break;
3558: if( !inPhrase &&
3559: pSegment[iEnd]==':' &&
3560: (iCol = checkColumnSpecifier(pQuery->pFts, pToken, nToken))>=0 ){
3561: pQuery->nextColumn = iCol;
3562: continue;
3563: }
3564: if( !inPhrase && pQuery->nTerms>0 && nToken==2
3565: && pSegment[iBegin]=='O' && pSegment[iBegin+1]=='R' ){
3566: pQuery->nextIsOr = 1;
3567: continue;
3568: }
3569: queryAdd(pQuery, pToken, nToken);
3570: if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){
3571: pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
3572: }
3573: if( iEnd<nSegment && pSegment[iEnd]=='*' ){
3574: pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
3575: }
3576: pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm;
3577: if( inPhrase ){
3578: nTerm++;
3579: }
3580: }
3581:
3582: if( inPhrase && pQuery->nTerms>firstIndex ){
3583: pQuery->pTerms[firstIndex].nPhrase = pQuery->nTerms - firstIndex - 1;
3584: }
3585:
3586: return pModule->xClose(pCursor);
3587: }
3588:
3589: /* Parse a query string, yielding a Query object pQuery.
3590: **
3591: ** The calling function will need to queryClear() to clean up
3592: ** the dynamically allocated memory held by pQuery.
3593: */
3594: static int parseQuery(
3595: fulltext_vtab *v, /* The fulltext index */
3596: const char *zInput, /* Input text of the query string */
3597: int nInput, /* Size of the input text */
3598: int dfltColumn, /* Default column of the index to match against */
3599: Query *pQuery /* Write the parse results here. */
3600: ){
3601: int iInput, inPhrase = 0;
3602:
3603: if( zInput==0 ) nInput = 0;
3604: if( nInput<0 ) nInput = strlen(zInput);
3605: pQuery->nTerms = 0;
3606: pQuery->pTerms = NULL;
3607: pQuery->nextIsOr = 0;
3608: pQuery->nextColumn = dfltColumn;
3609: pQuery->dfltColumn = dfltColumn;
3610: pQuery->pFts = v;
3611:
3612: for(iInput=0; iInput<nInput; ++iInput){
3613: int i;
3614: for(i=iInput; i<nInput && zInput[i]!='"'; ++i){}
3615: if( i>iInput ){
3616: tokenizeSegment(v->pTokenizer, zInput+iInput, i-iInput, inPhrase,
3617: pQuery);
3618: }
3619: iInput = i;
3620: if( i<nInput ){
3621: assert( zInput[i]=='"' );
3622: inPhrase = !inPhrase;
3623: }
3624: }
3625:
3626: if( inPhrase ){
3627: /* unmatched quote */
3628: queryClear(pQuery);
3629: return SQLITE_ERROR;
3630: }
3631: return SQLITE_OK;
3632: }
3633:
3634: /* TODO(shess) Refactor the code to remove this forward decl. */
3635: static int flushPendingTerms(fulltext_vtab *v);
3636:
3637: /* Perform a full-text query using the search expression in
3638: ** zInput[0..nInput-1]. Return a list of matching documents
3639: ** in pResult.
3640: **
3641: ** Queries must match column iColumn. Or if iColumn>=nColumn
3642: ** they are allowed to match against any column.
3643: */
3644: static int fulltextQuery(
3645: fulltext_vtab *v, /* The full text index */
3646: int iColumn, /* Match against this column by default */
3647: const char *zInput, /* The query string */
3648: int nInput, /* Number of bytes in zInput[] */
3649: DataBuffer *pResult, /* Write the result doclist here */
3650: Query *pQuery /* Put parsed query string here */
3651: ){
3652: int i, iNext, rc;
3653: DataBuffer left, right, or, new;
3654: int nNot = 0;
3655: QueryTerm *aTerm;
3656:
3657: /* TODO(shess) Instead of flushing pendingTerms, we could query for
3658: ** the relevant term and merge the doclist into what we receive from
3659: ** the database. Wait and see if this is a common issue, first.
3660: **
3661: ** A good reason not to flush is to not generate update-related
3662: ** error codes from here.
3663: */
3664:
3665: /* Flush any buffered updates before executing the query. */
3666: rc = flushPendingTerms(v);
3667: if( rc!=SQLITE_OK ) return rc;
3668:
3669: /* TODO(shess) I think that the queryClear() calls below are not
3670: ** necessary, because fulltextClose() already clears the query.
3671: */
3672: rc = parseQuery(v, zInput, nInput, iColumn, pQuery);
3673: if( rc!=SQLITE_OK ) return rc;
3674:
3675: /* Empty or NULL queries return no results. */
3676: if( pQuery->nTerms==0 ){
3677: dataBufferInit(pResult, 0);
3678: return SQLITE_OK;
3679: }
3680:
3681: /* Merge AND terms. */
3682: /* TODO(shess) I think we can early-exit if( i>nNot && left.nData==0 ). */
3683: aTerm = pQuery->pTerms;
3684: for(i = 0; i<pQuery->nTerms; i=iNext){
3685: if( aTerm[i].isNot ){
3686: /* Handle all NOT terms in a separate pass */
3687: nNot++;
3688: iNext = i + aTerm[i].nPhrase+1;
3689: continue;
3690: }
3691: iNext = i + aTerm[i].nPhrase + 1;
3692: rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
3693: if( rc ){
3694: if( i!=nNot ) dataBufferDestroy(&left);
3695: queryClear(pQuery);
3696: return rc;
3697: }
3698: while( iNext<pQuery->nTerms && aTerm[iNext].isOr ){
3699: rc = docListOfTerm(v, aTerm[iNext].iColumn, &aTerm[iNext], &or);
3700: iNext += aTerm[iNext].nPhrase + 1;
3701: if( rc ){
3702: if( i!=nNot ) dataBufferDestroy(&left);
3703: dataBufferDestroy(&right);
3704: queryClear(pQuery);
3705: return rc;
3706: }
3707: dataBufferInit(&new, 0);
3708: docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new);
3709: dataBufferDestroy(&right);
3710: dataBufferDestroy(&or);
3711: right = new;
3712: }
3713: if( i==nNot ){ /* first term processed. */
3714: left = right;
3715: }else{
3716: dataBufferInit(&new, 0);
3717: docListAndMerge(left.pData, left.nData, right.pData, right.nData, &new);
3718: dataBufferDestroy(&right);
3719: dataBufferDestroy(&left);
3720: left = new;
3721: }
3722: }
3723:
3724: if( nNot==pQuery->nTerms ){
3725: /* We do not yet know how to handle a query of only NOT terms */
3726: return SQLITE_ERROR;
3727: }
3728:
3729: /* Do the EXCEPT terms */
3730: for(i=0; i<pQuery->nTerms; i += aTerm[i].nPhrase + 1){
3731: if( !aTerm[i].isNot ) continue;
3732: rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
3733: if( rc ){
3734: queryClear(pQuery);
3735: dataBufferDestroy(&left);
3736: return rc;
3737: }
3738: dataBufferInit(&new, 0);
3739: docListExceptMerge(left.pData, left.nData, right.pData, right.nData, &new);
3740: dataBufferDestroy(&right);
3741: dataBufferDestroy(&left);
3742: left = new;
3743: }
3744:
3745: *pResult = left;
3746: return rc;
3747: }
3748:
3749: /*
3750: ** This is the xFilter interface for the virtual table. See
3751: ** the virtual table xFilter method documentation for additional
3752: ** information.
3753: **
3754: ** If idxNum==QUERY_GENERIC then do a full table scan against
3755: ** the %_content table.
3756: **
3757: ** If idxNum==QUERY_ROWID then do a rowid lookup for a single entry
3758: ** in the %_content table.
3759: **
3760: ** If idxNum>=QUERY_FULLTEXT then use the full text index. The
3761: ** column on the left-hand side of the MATCH operator is column
3762: ** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
3763: ** side of the MATCH operator.
3764: */
3765: /* TODO(shess) Upgrade the cursor initialization and destruction to
3766: ** account for fulltextFilter() being called multiple times on the
3767: ** same cursor. The current solution is very fragile. Apply fix to
3768: ** fts2 as appropriate.
3769: */
3770: static int fulltextFilter(
3771: sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
3772: int idxNum, const char *idxStr, /* Which indexing scheme to use */
3773: int argc, sqlite3_value **argv /* Arguments for the indexing scheme */
3774: ){
3775: fulltext_cursor *c = (fulltext_cursor *) pCursor;
3776: fulltext_vtab *v = cursor_vtab(c);
3777: int rc;
3778:
3779: TRACE(("FTS2 Filter %p\n",pCursor));
3780:
3781: /* If the cursor has a statement that was not prepared according to
3782: ** idxNum, clear it. I believe all calls to fulltextFilter with a
3783: ** given cursor will have the same idxNum , but in this case it's
3784: ** easy to be safe.
3785: */
3786: if( c->pStmt && c->iCursorType!=idxNum ){
3787: sqlite3_finalize(c->pStmt);
3788: c->pStmt = NULL;
3789: }
3790:
3791: /* Get a fresh statement appropriate to idxNum. */
3792: /* TODO(shess): Add a prepared-statement cache in the vt structure.
3793: ** The cache must handle multiple open cursors. Easier to cache the
3794: ** statement variants at the vt to reduce malloc/realloc/free here.
3795: ** Or we could have a StringBuffer variant which allowed stack
3796: ** construction for small values.
3797: */
3798: if( !c->pStmt ){
3799: char *zSql = sqlite3_mprintf("select rowid, * from %%_content %s",
3800: idxNum==QUERY_GENERIC ? "" : "where rowid=?");
3801: rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, zSql);
3802: sqlite3_free(zSql);
3803: if( rc!=SQLITE_OK ) return rc;
3804: c->iCursorType = idxNum;
3805: }else{
3806: sqlite3_reset(c->pStmt);
3807: assert( c->iCursorType==idxNum );
3808: }
3809:
3810: switch( idxNum ){
3811: case QUERY_GENERIC:
3812: break;
3813:
3814: case QUERY_ROWID:
3815: rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
3816: if( rc!=SQLITE_OK ) return rc;
3817: break;
3818:
3819: default: /* full-text search */
3820: {
3821: const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
3822: assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
3823: assert( argc==1 );
3824: queryClear(&c->q);
3825: if( c->result.nData!=0 ){
3826: /* This case happens if the same cursor is used repeatedly. */
3827: dlrDestroy(&c->reader);
3828: dataBufferReset(&c->result);
3829: }else{
3830: dataBufferInit(&c->result, 0);
3831: }
3832: rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &c->result, &c->q);
3833: if( rc!=SQLITE_OK ) return rc;
3834: if( c->result.nData!=0 ){
3835: dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
3836: }
3837: break;
3838: }
3839: }
3840:
3841: return fulltextNext(pCursor);
3842: }
3843:
3844: /* This is the xEof method of the virtual table. The SQLite core
3845: ** calls this routine to find out if it has reached the end of
3846: ** a query's results set.
3847: */
3848: static int fulltextEof(sqlite3_vtab_cursor *pCursor){
3849: fulltext_cursor *c = (fulltext_cursor *) pCursor;
3850: return c->eof;
3851: }
3852:
3853: /* This is the xColumn method of the virtual table. The SQLite
3854: ** core calls this method during a query when it needs the value
3855: ** of a column from the virtual table. This method needs to use
3856: ** one of the sqlite3_result_*() routines to store the requested
3857: ** value back in the pContext.
3858: */
3859: static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
3860: sqlite3_context *pContext, int idxCol){
3861: fulltext_cursor *c = (fulltext_cursor *) pCursor;
3862: fulltext_vtab *v = cursor_vtab(c);
3863:
3864: if( idxCol<v->nColumn ){
3865: sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
3866: sqlite3_result_value(pContext, pVal);
3867: }else if( idxCol==v->nColumn ){
3868: /* The extra column whose name is the same as the table.
3869: ** Return a blob which is a pointer to the cursor
3870: */
3871: sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
3872: }
3873: return SQLITE_OK;
3874: }
3875:
3876: /* This is the xRowid method. The SQLite core calls this routine to
3877: ** retrive the rowid for the current row of the result set. The
3878: ** rowid should be written to *pRowid.
3879: */
3880: static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
3881: fulltext_cursor *c = (fulltext_cursor *) pCursor;
3882:
3883: *pRowid = sqlite3_column_int64(c->pStmt, 0);
3884: return SQLITE_OK;
3885: }
3886:
3887: /* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0,
3888: ** we also store positions and offsets in the hash table using that
3889: ** column number.
3890: */
3891: static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
3892: const char *zText, int iColumn){
3893: sqlite3_tokenizer *pTokenizer = v->pTokenizer;
3894: sqlite3_tokenizer_cursor *pCursor;
3895: const char *pToken;
3896: int nTokenBytes;
3897: int iStartOffset, iEndOffset, iPosition;
3898: int rc;
3899:
3900: rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
3901: if( rc!=SQLITE_OK ) return rc;
3902:
3903: pCursor->pTokenizer = pTokenizer;
3904: while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
3905: &pToken, &nTokenBytes,
3906: &iStartOffset, &iEndOffset,
3907: &iPosition)) ){
3908: DLCollector *p;
3909: int nData; /* Size of doclist before our update. */
3910:
3911: /* Positions can't be negative; we use -1 as a terminator
3912: * internally. Token can't be NULL or empty. */
3913: if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){
3914: rc = SQLITE_ERROR;
3915: break;
3916: }
3917:
3918: p = fts2HashFind(&v->pendingTerms, pToken, nTokenBytes);
3919: if( p==NULL ){
3920: nData = 0;
3921: p = dlcNew(iDocid, DL_DEFAULT);
3922: fts2HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
3923:
3924: /* Overhead for our hash table entry, the key, and the value. */
3925: v->nPendingData += sizeof(struct fts2HashElem)+sizeof(*p)+nTokenBytes;
3926: }else{
3927: nData = p->b.nData;
3928: if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
3929: }
3930: if( iColumn>=0 ){
3931: dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
3932: }
3933:
3934: /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
3935: v->nPendingData += p->b.nData-nData;
3936: }
3937:
3938: /* TODO(shess) Check return? Should this be able to cause errors at
3939: ** this point? Actually, same question about sqlite3_finalize(),
3940: ** though one could argue that failure there means that the data is
3941: ** not durable. *ponder*
3942: */
3943: pTokenizer->pModule->xClose(pCursor);
3944: if( SQLITE_DONE == rc ) return SQLITE_OK;
3945: return rc;
3946: }
3947:
3948: /* Add doclists for all terms in [pValues] to pendingTerms table. */
3949: static int insertTerms(fulltext_vtab *v, sqlite_int64 iRowid,
3950: sqlite3_value **pValues){
3951: int i;
3952: for(i = 0; i < v->nColumn ; ++i){
3953: char *zText = (char*)sqlite3_value_text(pValues[i]);
3954: int rc = buildTerms(v, iRowid, zText, i);
3955: if( rc!=SQLITE_OK ) return rc;
3956: }
3957: return SQLITE_OK;
3958: }
3959:
3960: /* Add empty doclists for all terms in the given row's content to
3961: ** pendingTerms.
3962: */
3963: static int deleteTerms(fulltext_vtab *v, sqlite_int64 iRowid){
3964: const char **pValues;
3965: int i, rc;
3966:
3967: /* TODO(shess) Should we allow such tables at all? */
3968: if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
3969:
3970: rc = content_select(v, iRowid, &pValues);
3971: if( rc!=SQLITE_OK ) return rc;
3972:
3973: for(i = 0 ; i < v->nColumn; ++i) {
3974: rc = buildTerms(v, iRowid, pValues[i], -1);
3975: if( rc!=SQLITE_OK ) break;
3976: }
3977:
3978: freeStringArray(v->nColumn, pValues);
3979: return SQLITE_OK;
3980: }
3981:
3982: /* TODO(shess) Refactor the code to remove this forward decl. */
3983: static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
3984:
3985: /* Insert a row into the %_content table; set *piRowid to be the ID of the
3986: ** new row. Add doclists for terms to pendingTerms.
3987: */
3988: static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestRowid,
3989: sqlite3_value **pValues, sqlite_int64 *piRowid){
3990: int rc;
3991:
3992: rc = content_insert(v, pRequestRowid, pValues); /* execute an SQL INSERT */
3993: if( rc!=SQLITE_OK ) return rc;
3994:
3995: *piRowid = sqlite3_last_insert_rowid(v->db);
3996: rc = initPendingTerms(v, *piRowid);
3997: if( rc!=SQLITE_OK ) return rc;
3998:
3999: return insertTerms(v, *piRowid, pValues);
4000: }
4001:
4002: /* Delete a row from the %_content table; add empty doclists for terms
4003: ** to pendingTerms.
4004: */
4005: static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
4006: int rc = initPendingTerms(v, iRow);
4007: if( rc!=SQLITE_OK ) return rc;
4008:
4009: rc = deleteTerms(v, iRow);
4010: if( rc!=SQLITE_OK ) return rc;
4011:
4012: return content_delete(v, iRow); /* execute an SQL DELETE */
4013: }
4014:
4015: /* Update a row in the %_content table; add delete doclists to
4016: ** pendingTerms for old terms not in the new data, add insert doclists
4017: ** to pendingTerms for terms in the new data.
4018: */
4019: static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
4020: sqlite3_value **pValues){
4021: int rc = initPendingTerms(v, iRow);
4022: if( rc!=SQLITE_OK ) return rc;
4023:
4024: /* Generate an empty doclist for each term that previously appeared in this
4025: * row. */
4026: rc = deleteTerms(v, iRow);
4027: if( rc!=SQLITE_OK ) return rc;
4028:
4029: rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */
4030: if( rc!=SQLITE_OK ) return rc;
4031:
4032: /* Now add positions for terms which appear in the updated row. */
4033: return insertTerms(v, iRow, pValues);
4034: }
4035:
4036: /*******************************************************************/
4037: /* InteriorWriter is used to collect terms and block references into
4038: ** interior nodes in %_segments. See commentary at top of file for
4039: ** format.
4040: */
4041:
4042: /* How large interior nodes can grow. */
4043: #define INTERIOR_MAX 2048
4044:
4045: /* Minimum number of terms per interior node (except the root). This
4046: ** prevents large terms from making the tree too skinny - must be >0
4047: ** so that the tree always makes progress. Note that the min tree
4048: ** fanout will be INTERIOR_MIN_TERMS+1.
4049: */
4050: #define INTERIOR_MIN_TERMS 7
4051: #if INTERIOR_MIN_TERMS<1
4052: # error INTERIOR_MIN_TERMS must be greater than 0.
4053: #endif
4054:
4055: /* ROOT_MAX controls how much data is stored inline in the segment
4056: ** directory.
4057: */
4058: /* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's
4059: ** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
4060: ** can both see it, but if the caller passed it in, we wouldn't even
4061: ** need a define.
4062: */
4063: #define ROOT_MAX 1024
4064: #if ROOT_MAX<VARINT_MAX*2
4065: # error ROOT_MAX must have enough space for a header.
4066: #endif
4067:
4068: /* InteriorBlock stores a linked-list of interior blocks while a lower
4069: ** layer is being constructed.
4070: */
4071: typedef struct InteriorBlock {
4072: DataBuffer term; /* Leftmost term in block's subtree. */
4073: DataBuffer data; /* Accumulated data for the block. */
4074: struct InteriorBlock *next;
4075: } InteriorBlock;
4076:
4077: static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
4078: const char *pTerm, int nTerm){
4079: InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
4080: char c[VARINT_MAX+VARINT_MAX];
4081: int n;
4082:
4083: if( block ){
4084: memset(block, 0, sizeof(*block));
4085: dataBufferInit(&block->term, 0);
4086: dataBufferReplace(&block->term, pTerm, nTerm);
4087:
4088: n = putVarint(c, iHeight);
4089: n += putVarint(c+n, iChildBlock);
4090: dataBufferInit(&block->data, INTERIOR_MAX);
4091: dataBufferReplace(&block->data, c, n);
4092: }
4093: return block;
4094: }
4095:
4096: #ifndef NDEBUG
4097: /* Verify that the data is readable as an interior node. */
4098: static void interiorBlockValidate(InteriorBlock *pBlock){
4099: const char *pData = pBlock->data.pData;
4100: int nData = pBlock->data.nData;
4101: int n, iDummy;
4102: sqlite_int64 iBlockid;
4103:
4104: assert( nData>0 );
4105: assert( pData!=0 );
4106: assert( pData+nData>pData );
4107:
4108: /* Must lead with height of node as a varint(n), n>0 */
4109: n = getVarint32(pData, &iDummy);
4110: assert( n>0 );
4111: assert( iDummy>0 );
4112: assert( n<nData );
4113: pData += n;
4114: nData -= n;
4115:
4116: /* Must contain iBlockid. */
4117: n = getVarint(pData, &iBlockid);
4118: assert( n>0 );
4119: assert( n<=nData );
4120: pData += n;
4121: nData -= n;
4122:
4123: /* Zero or more terms of positive length */
4124: if( nData!=0 ){
4125: /* First term is not delta-encoded. */
4126: n = getVarint32(pData, &iDummy);
4127: assert( n>0 );
4128: assert( iDummy>0 );
4129: assert( n+iDummy>0);
4130: assert( n+iDummy<=nData );
4131: pData += n+iDummy;
4132: nData -= n+iDummy;
4133:
4134: /* Following terms delta-encoded. */
4135: while( nData!=0 ){
4136: /* Length of shared prefix. */
4137: n = getVarint32(pData, &iDummy);
4138: assert( n>0 );
4139: assert( iDummy>=0 );
4140: assert( n<nData );
4141: pData += n;
4142: nData -= n;
4143:
4144: /* Length and data of distinct suffix. */
4145: n = getVarint32(pData, &iDummy);
4146: assert( n>0 );
4147: assert( iDummy>0 );
4148: assert( n+iDummy>0);
4149: assert( n+iDummy<=nData );
4150: pData += n+iDummy;
4151: nData -= n+iDummy;
4152: }
4153: }
4154: }
4155: #define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
4156: #else
4157: #define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
4158: #endif
4159:
4160: typedef struct InteriorWriter {
4161: int iHeight; /* from 0 at leaves. */
4162: InteriorBlock *first, *last;
4163: struct InteriorWriter *parentWriter;
4164:
4165: DataBuffer term; /* Last term written to block "last". */
4166: sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
4167: #ifndef NDEBUG
4168: sqlite_int64 iLastChildBlock; /* for consistency checks. */
4169: #endif
4170: } InteriorWriter;
4171:
4172: /* Initialize an interior node where pTerm[nTerm] marks the leftmost
4173: ** term in the tree. iChildBlock is the leftmost child block at the
4174: ** next level down the tree.
4175: */
4176: static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
4177: sqlite_int64 iChildBlock,
4178: InteriorWriter *pWriter){
4179: InteriorBlock *block;
4180: assert( iHeight>0 );
4181: CLEAR(pWriter);
4182:
4183: pWriter->iHeight = iHeight;
4184: pWriter->iOpeningChildBlock = iChildBlock;
4185: #ifndef NDEBUG
4186: pWriter->iLastChildBlock = iChildBlock;
4187: #endif
4188: block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
4189: pWriter->last = pWriter->first = block;
4190: ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
4191: dataBufferInit(&pWriter->term, 0);
4192: }
4193:
4194: /* Append the child node rooted at iChildBlock to the interior node,
4195: ** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
4196: */
4197: static void interiorWriterAppend(InteriorWriter *pWriter,
4198: const char *pTerm, int nTerm,
4199: sqlite_int64 iChildBlock){
4200: char c[VARINT_MAX+VARINT_MAX];
4201: int n, nPrefix = 0;
4202:
4203: ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
4204:
4205: /* The first term written into an interior node is actually
4206: ** associated with the second child added (the first child was added
4207: ** in interiorWriterInit, or in the if clause at the bottom of this
4208: ** function). That term gets encoded straight up, with nPrefix left
4209: ** at 0.
4210: */
4211: if( pWriter->term.nData==0 ){
4212: n = putVarint(c, nTerm);
4213: }else{
4214: while( nPrefix<pWriter->term.nData &&
4215: pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
4216: nPrefix++;
4217: }
4218:
4219: n = putVarint(c, nPrefix);
4220: n += putVarint(c+n, nTerm-nPrefix);
4221: }
4222:
4223: #ifndef NDEBUG
4224: pWriter->iLastChildBlock++;
4225: #endif
4226: assert( pWriter->iLastChildBlock==iChildBlock );
4227:
4228: /* Overflow to a new block if the new term makes the current block
4229: ** too big, and the current block already has enough terms.
4230: */
4231: if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
4232: iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
4233: pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
4234: pTerm, nTerm);
4235: pWriter->last = pWriter->last->next;
4236: pWriter->iOpeningChildBlock = iChildBlock;
4237: dataBufferReset(&pWriter->term);
4238: }else{
4239: dataBufferAppend2(&pWriter->last->data, c, n,
4240: pTerm+nPrefix, nTerm-nPrefix);
4241: dataBufferReplace(&pWriter->term, pTerm, nTerm);
4242: }
4243: ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
4244: }
4245:
4246: /* Free the space used by pWriter, including the linked-list of
4247: ** InteriorBlocks, and parentWriter, if present.
4248: */
4249: static int interiorWriterDestroy(InteriorWriter *pWriter){
4250: InteriorBlock *block = pWriter->first;
4251:
4252: while( block!=NULL ){
4253: InteriorBlock *b = block;
4254: block = block->next;
4255: dataBufferDestroy(&b->term);
4256: dataBufferDestroy(&b->data);
4257: sqlite3_free(b);
4258: }
4259: if( pWriter->parentWriter!=NULL ){
4260: interiorWriterDestroy(pWriter->parentWriter);
4261: sqlite3_free(pWriter->parentWriter);
4262: }
4263: dataBufferDestroy(&pWriter->term);
4264: SCRAMBLE(pWriter);
4265: return SQLITE_OK;
4266: }
4267:
4268: /* If pWriter can fit entirely in ROOT_MAX, return it as the root info
4269: ** directly, leaving *piEndBlockid unchanged. Otherwise, flush
4270: ** pWriter to %_segments, building a new layer of interior nodes, and
4271: ** recursively ask for their root into.
4272: */
4273: static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter,
4274: char **ppRootInfo, int *pnRootInfo,
4275: sqlite_int64 *piEndBlockid){
4276: InteriorBlock *block = pWriter->first;
4277: sqlite_int64 iBlockid = 0;
4278: int rc;
4279:
4280: /* If we can fit the segment inline */
4281: if( block==pWriter->last && block->data.nData<ROOT_MAX ){
4282: *ppRootInfo = block->data.pData;
4283: *pnRootInfo = block->data.nData;
4284: return SQLITE_OK;
4285: }
4286:
4287: /* Flush the first block to %_segments, and create a new level of
4288: ** interior node.
4289: */
4290: ASSERT_VALID_INTERIOR_BLOCK(block);
4291: rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
4292: if( rc!=SQLITE_OK ) return rc;
4293: *piEndBlockid = iBlockid;
4294:
4295: pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
4296: interiorWriterInit(pWriter->iHeight+1,
4297: block->term.pData, block->term.nData,
4298: iBlockid, pWriter->parentWriter);
4299:
4300: /* Flush additional blocks and append to the higher interior
4301: ** node.
4302: */
4303: for(block=block->next; block!=NULL; block=block->next){
4304: ASSERT_VALID_INTERIOR_BLOCK(block);
4305: rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
4306: if( rc!=SQLITE_OK ) return rc;
4307: *piEndBlockid = iBlockid;
4308:
4309: interiorWriterAppend(pWriter->parentWriter,
4310: block->term.pData, block->term.nData, iBlockid);
4311: }
4312:
4313: /* Parent node gets the chance to be the root. */
4314: return interiorWriterRootInfo(v, pWriter->parentWriter,
4315: ppRootInfo, pnRootInfo, piEndBlockid);
4316: }
4317:
4318: /****************************************************************/
4319: /* InteriorReader is used to read off the data from an interior node
4320: ** (see comment at top of file for the format).
4321: */
4322: typedef struct InteriorReader {
4323: const char *pData;
4324: int nData;
4325:
4326: DataBuffer term; /* previous term, for decoding term delta. */
4327:
4328: sqlite_int64 iBlockid;
4329: } InteriorReader;
4330:
4331: static void interiorReaderDestroy(InteriorReader *pReader){
4332: dataBufferDestroy(&pReader->term);
4333: SCRAMBLE(pReader);
4334: }
4335:
4336: /* TODO(shess) The assertions are great, but what if we're in NDEBUG
4337: ** and the blob is empty or otherwise contains suspect data?
4338: */
4339: static void interiorReaderInit(const char *pData, int nData,
4340: InteriorReader *pReader){
4341: int n, nTerm;
4342:
4343: /* Require at least the leading flag byte */
4344: assert( nData>0 );
4345: assert( pData[0]!='\0' );
4346:
4347: CLEAR(pReader);
4348:
4349: /* Decode the base blockid, and set the cursor to the first term. */
4350: n = getVarint(pData+1, &pReader->iBlockid);
4351: assert( 1+n<=nData );
4352: pReader->pData = pData+1+n;
4353: pReader->nData = nData-(1+n);
4354:
4355: /* A single-child interior node (such as when a leaf node was too
4356: ** large for the segment directory) won't have any terms.
4357: ** Otherwise, decode the first term.
4358: */
4359: if( pReader->nData==0 ){
4360: dataBufferInit(&pReader->term, 0);
4361: }else{
4362: n = getVarint32(pReader->pData, &nTerm);
4363: dataBufferInit(&pReader->term, nTerm);
4364: dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
4365: assert( n+nTerm<=pReader->nData );
4366: pReader->pData += n+nTerm;
4367: pReader->nData -= n+nTerm;
4368: }
4369: }
4370:
4371: static int interiorReaderAtEnd(InteriorReader *pReader){
4372: return pReader->term.nData==0;
4373: }
4374:
4375: static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
4376: return pReader->iBlockid;
4377: }
4378:
4379: static int interiorReaderTermBytes(InteriorReader *pReader){
4380: assert( !interiorReaderAtEnd(pReader) );
4381: return pReader->term.nData;
4382: }
4383: static const char *interiorReaderTerm(InteriorReader *pReader){
4384: assert( !interiorReaderAtEnd(pReader) );
4385: return pReader->term.pData;
4386: }
4387:
4388: /* Step forward to the next term in the node. */
4389: static void interiorReaderStep(InteriorReader *pReader){
4390: assert( !interiorReaderAtEnd(pReader) );
4391:
4392: /* If the last term has been read, signal eof, else construct the
4393: ** next term.
4394: */
4395: if( pReader->nData==0 ){
4396: dataBufferReset(&pReader->term);
4397: }else{
4398: int n, nPrefix, nSuffix;
4399:
4400: n = getVarint32(pReader->pData, &nPrefix);
4401: n += getVarint32(pReader->pData+n, &nSuffix);
4402:
4403: /* Truncate the current term and append suffix data. */
4404: pReader->term.nData = nPrefix;
4405: dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
4406:
4407: assert( n+nSuffix<=pReader->nData );
4408: pReader->pData += n+nSuffix;
4409: pReader->nData -= n+nSuffix;
4410: }
4411: pReader->iBlockid++;
4412: }
4413:
4414: /* Compare the current term to pTerm[nTerm], returning strcmp-style
4415: ** results. If isPrefix, equality means equal through nTerm bytes.
4416: */
4417: static int interiorReaderTermCmp(InteriorReader *pReader,
4418: const char *pTerm, int nTerm, int isPrefix){
4419: const char *pReaderTerm = interiorReaderTerm(pReader);
4420: int nReaderTerm = interiorReaderTermBytes(pReader);
4421: int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
4422:
4423: if( n==0 ){
4424: if( nReaderTerm>0 ) return -1;
4425: if( nTerm>0 ) return 1;
4426: return 0;
4427: }
4428:
4429: c = memcmp(pReaderTerm, pTerm, n);
4430: if( c!=0 ) return c;
4431: if( isPrefix && n==nTerm ) return 0;
4432: return nReaderTerm - nTerm;
4433: }
4434:
4435: /****************************************************************/
4436: /* LeafWriter is used to collect terms and associated doclist data
4437: ** into leaf blocks in %_segments (see top of file for format info).
4438: ** Expected usage is:
4439: **
4440: ** LeafWriter writer;
4441: ** leafWriterInit(0, 0, &writer);
4442: ** while( sorted_terms_left_to_process ){
4443: ** // data is doclist data for that term.
4444: ** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
4445: ** if( rc!=SQLITE_OK ) goto err;
4446: ** }
4447: ** rc = leafWriterFinalize(v, &writer);
4448: **err:
4449: ** leafWriterDestroy(&writer);
4450: ** return rc;
4451: **
4452: ** leafWriterStep() may write a collected leaf out to %_segments.
4453: ** leafWriterFinalize() finishes writing any buffered data and stores
4454: ** a root node in %_segdir. leafWriterDestroy() frees all buffers and
4455: ** InteriorWriters allocated as part of writing this segment.
4456: **
4457: ** TODO(shess) Document leafWriterStepMerge().
4458: */
4459:
4460: /* Put terms with data this big in their own block. */
4461: #define STANDALONE_MIN 1024
4462:
4463: /* Keep leaf blocks below this size. */
4464: #define LEAF_MAX 2048
4465:
4466: typedef struct LeafWriter {
4467: int iLevel;
4468: int idx;
4469: sqlite_int64 iStartBlockid; /* needed to create the root info */
4470: sqlite_int64 iEndBlockid; /* when we're done writing. */
4471:
4472: DataBuffer term; /* previous encoded term */
4473: DataBuffer data; /* encoding buffer */
4474:
4475: /* bytes of first term in the current node which distinguishes that
4476: ** term from the last term of the previous node.
4477: */
4478: int nTermDistinct;
4479:
4480: InteriorWriter parentWriter; /* if we overflow */
4481: int has_parent;
4482: } LeafWriter;
4483:
4484: static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
4485: CLEAR(pWriter);
4486: pWriter->iLevel = iLevel;
4487: pWriter->idx = idx;
4488:
4489: dataBufferInit(&pWriter->term, 32);
4490:
4491: /* Start out with a reasonably sized block, though it can grow. */
4492: dataBufferInit(&pWriter->data, LEAF_MAX);
4493: }
4494:
4495: #ifndef NDEBUG
4496: /* Verify that the data is readable as a leaf node. */
4497: static void leafNodeValidate(const char *pData, int nData){
4498: int n, iDummy;
4499:
4500: if( nData==0 ) return;
4501: assert( nData>0 );
4502: assert( pData!=0 );
4503: assert( pData+nData>pData );
4504:
4505: /* Must lead with a varint(0) */
4506: n = getVarint32(pData, &iDummy);
4507: assert( iDummy==0 );
4508: assert( n>0 );
4509: assert( n<nData );
4510: pData += n;
4511: nData -= n;
4512:
4513: /* Leading term length and data must fit in buffer. */
4514: n = getVarint32(pData, &iDummy);
4515: assert( n>0 );
4516: assert( iDummy>0 );
4517: assert( n+iDummy>0 );
4518: assert( n+iDummy<nData );
4519: pData += n+iDummy;
4520: nData -= n+iDummy;
4521:
4522: /* Leading term's doclist length and data must fit. */
4523: n = getVarint32(pData, &iDummy);
4524: assert( n>0 );
4525: assert( iDummy>0 );
4526: assert( n+iDummy>0 );
4527: assert( n+iDummy<=nData );
4528: ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
4529: pData += n+iDummy;
4530: nData -= n+iDummy;
4531:
4532: /* Verify that trailing terms and doclists also are readable. */
4533: while( nData!=0 ){
4534: n = getVarint32(pData, &iDummy);
4535: assert( n>0 );
4536: assert( iDummy>=0 );
4537: assert( n<nData );
4538: pData += n;
4539: nData -= n;
4540: n = getVarint32(pData, &iDummy);
4541: assert( n>0 );
4542: assert( iDummy>0 );
4543: assert( n+iDummy>0 );
4544: assert( n+iDummy<nData );
4545: pData += n+iDummy;
4546: nData -= n+iDummy;
4547:
4548: n = getVarint32(pData, &iDummy);
4549: assert( n>0 );
4550: assert( iDummy>0 );
4551: assert( n+iDummy>0 );
4552: assert( n+iDummy<=nData );
4553: ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
4554: pData += n+iDummy;
4555: nData -= n+iDummy;
4556: }
4557: }
4558: #define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
4559: #else
4560: #define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
4561: #endif
4562:
4563: /* Flush the current leaf node to %_segments, and adding the resulting
4564: ** blockid and the starting term to the interior node which will
4565: ** contain it.
4566: */
4567: static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter,
4568: int iData, int nData){
4569: sqlite_int64 iBlockid = 0;
4570: const char *pStartingTerm;
4571: int nStartingTerm, rc, n;
4572:
4573: /* Must have the leading varint(0) flag, plus at least some
4574: ** valid-looking data.
4575: */
4576: assert( nData>2 );
4577: assert( iData>=0 );
4578: assert( iData+nData<=pWriter->data.nData );
4579: ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
4580:
4581: rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
4582: if( rc!=SQLITE_OK ) return rc;
4583: assert( iBlockid!=0 );
4584:
4585: /* Reconstruct the first term in the leaf for purposes of building
4586: ** the interior node.
4587: */
4588: n = getVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
4589: pStartingTerm = pWriter->data.pData+iData+1+n;
4590: assert( pWriter->data.nData>iData+1+n+nStartingTerm );
4591: assert( pWriter->nTermDistinct>0 );
4592: assert( pWriter->nTermDistinct<=nStartingTerm );
4593: nStartingTerm = pWriter->nTermDistinct;
4594:
4595: if( pWriter->has_parent ){
4596: interiorWriterAppend(&pWriter->parentWriter,
4597: pStartingTerm, nStartingTerm, iBlockid);
4598: }else{
4599: interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
4600: &pWriter->parentWriter);
4601: pWriter->has_parent = 1;
4602: }
4603:
4604: /* Track the span of this segment's leaf nodes. */
4605: if( pWriter->iEndBlockid==0 ){
4606: pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
4607: }else{
4608: pWriter->iEndBlockid++;
4609: assert( iBlockid==pWriter->iEndBlockid );
4610: }
4611:
4612: return SQLITE_OK;
4613: }
4614: static int leafWriterFlush(fulltext_vtab *v, LeafWriter *pWriter){
4615: int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
4616: if( rc!=SQLITE_OK ) return rc;
4617:
4618: /* Re-initialize the output buffer. */
4619: dataBufferReset(&pWriter->data);
4620:
4621: return SQLITE_OK;
4622: }
4623:
4624: /* Fetch the root info for the segment. If the entire leaf fits
4625: ** within ROOT_MAX, then it will be returned directly, otherwise it
4626: ** will be flushed and the root info will be returned from the
4627: ** interior node. *piEndBlockid is set to the blockid of the last
4628: ** interior or leaf node written to disk (0 if none are written at
4629: ** all).
4630: */
4631: static int leafWriterRootInfo(fulltext_vtab *v, LeafWriter *pWriter,
4632: char **ppRootInfo, int *pnRootInfo,
4633: sqlite_int64 *piEndBlockid){
4634: /* we can fit the segment entirely inline */
4635: if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
4636: *ppRootInfo = pWriter->data.pData;
4637: *pnRootInfo = pWriter->data.nData;
4638: *piEndBlockid = 0;
4639: return SQLITE_OK;
4640: }
4641:
4642: /* Flush remaining leaf data. */
4643: if( pWriter->data.nData>0 ){
4644: int rc = leafWriterFlush(v, pWriter);
4645: if( rc!=SQLITE_OK ) return rc;
4646: }
4647:
4648: /* We must have flushed a leaf at some point. */
4649: assert( pWriter->has_parent );
4650:
4651: /* Tenatively set the end leaf blockid as the end blockid. If the
4652: ** interior node can be returned inline, this will be the final
4653: ** blockid, otherwise it will be overwritten by
4654: ** interiorWriterRootInfo().
4655: */
4656: *piEndBlockid = pWriter->iEndBlockid;
4657:
4658: return interiorWriterRootInfo(v, &pWriter->parentWriter,
4659: ppRootInfo, pnRootInfo, piEndBlockid);
4660: }
4661:
4662: /* Collect the rootInfo data and store it into the segment directory.
4663: ** This has the effect of flushing the segment's leaf data to
4664: ** %_segments, and also flushing any interior nodes to %_segments.
4665: */
4666: static int leafWriterFinalize(fulltext_vtab *v, LeafWriter *pWriter){
4667: sqlite_int64 iEndBlockid;
4668: char *pRootInfo;
4669: int rc, nRootInfo;
4670:
4671: rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
4672: if( rc!=SQLITE_OK ) return rc;
4673:
4674: /* Don't bother storing an entirely empty segment. */
4675: if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
4676:
4677: return segdir_set(v, pWriter->iLevel, pWriter->idx,
4678: pWriter->iStartBlockid, pWriter->iEndBlockid,
4679: iEndBlockid, pRootInfo, nRootInfo);
4680: }
4681:
4682: static void leafWriterDestroy(LeafWriter *pWriter){
4683: if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
4684: dataBufferDestroy(&pWriter->term);
4685: dataBufferDestroy(&pWriter->data);
4686: }
4687:
4688: /* Encode a term into the leafWriter, delta-encoding as appropriate.
4689: ** Returns the length of the new term which distinguishes it from the
4690: ** previous term, which can be used to set nTermDistinct when a node
4691: ** boundary is crossed.
4692: */
4693: static int leafWriterEncodeTerm(LeafWriter *pWriter,
4694: const char *pTerm, int nTerm){
4695: char c[VARINT_MAX+VARINT_MAX];
4696: int n, nPrefix = 0;
4697:
4698: assert( nTerm>0 );
4699: while( nPrefix<pWriter->term.nData &&
4700: pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
4701: nPrefix++;
4702: /* Failing this implies that the terms weren't in order. */
4703: assert( nPrefix<nTerm );
4704: }
4705:
4706: if( pWriter->data.nData==0 ){
4707: /* Encode the node header and leading term as:
4708: ** varint(0)
4709: ** varint(nTerm)
4710: ** char pTerm[nTerm]
4711: */
4712: n = putVarint(c, '\0');
4713: n += putVarint(c+n, nTerm);
4714: dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
4715: }else{
4716: /* Delta-encode the term as:
4717: ** varint(nPrefix)
4718: ** varint(nSuffix)
4719: ** char pTermSuffix[nSuffix]
4720: */
4721: n = putVarint(c, nPrefix);
4722: n += putVarint(c+n, nTerm-nPrefix);
4723: dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
4724: }
4725: dataBufferReplace(&pWriter->term, pTerm, nTerm);
4726:
4727: return nPrefix+1;
4728: }
4729:
4730: /* Used to avoid a memmove when a large amount of doclist data is in
4731: ** the buffer. This constructs a node and term header before
4732: ** iDoclistData and flushes the resulting complete node using
4733: ** leafWriterInternalFlush().
4734: */
4735: static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter,
4736: const char *pTerm, int nTerm,
4737: int iDoclistData){
4738: char c[VARINT_MAX+VARINT_MAX];
4739: int iData, n = putVarint(c, 0);
4740: n += putVarint(c+n, nTerm);
4741:
4742: /* There should always be room for the header. Even if pTerm shared
4743: ** a substantial prefix with the previous term, the entire prefix
4744: ** could be constructed from earlier data in the doclist, so there
4745: ** should be room.
4746: */
4747: assert( iDoclistData>=n+nTerm );
4748:
4749: iData = iDoclistData-(n+nTerm);
4750: memcpy(pWriter->data.pData+iData, c, n);
4751: memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
4752:
4753: return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
4754: }
4755:
4756: /* Push pTerm[nTerm] along with the doclist data to the leaf layer of
4757: ** %_segments.
4758: */
4759: static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter,
4760: const char *pTerm, int nTerm,
4761: DLReader *pReaders, int nReaders){
4762: char c[VARINT_MAX+VARINT_MAX];
4763: int iTermData = pWriter->data.nData, iDoclistData;
4764: int i, nData, n, nActualData, nActual, rc, nTermDistinct;
4765:
4766: ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
4767: nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
4768:
4769: /* Remember nTermDistinct if opening a new node. */
4770: if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
4771:
4772: iDoclistData = pWriter->data.nData;
4773:
4774: /* Estimate the length of the merged doclist so we can leave space
4775: ** to encode it.
4776: */
4777: for(i=0, nData=0; i<nReaders; i++){
4778: nData += dlrAllDataBytes(&pReaders[i]);
4779: }
4780: n = putVarint(c, nData);
4781: dataBufferAppend(&pWriter->data, c, n);
4782:
4783: docListMerge(&pWriter->data, pReaders, nReaders);
4784: ASSERT_VALID_DOCLIST(DL_DEFAULT,
4785: pWriter->data.pData+iDoclistData+n,
4786: pWriter->data.nData-iDoclistData-n, NULL);
4787:
4788: /* The actual amount of doclist data at this point could be smaller
4789: ** than the length we encoded. Additionally, the space required to
4790: ** encode this length could be smaller. For small doclists, this is
4791: ** not a big deal, we can just use memmove() to adjust things.
4792: */
4793: nActualData = pWriter->data.nData-(iDoclistData+n);
4794: nActual = putVarint(c, nActualData);
4795: assert( nActualData<=nData );
4796: assert( nActual<=n );
4797:
4798: /* If the new doclist is big enough for force a standalone leaf
4799: ** node, we can immediately flush it inline without doing the
4800: ** memmove().
4801: */
4802: /* TODO(shess) This test matches leafWriterStep(), which does this
4803: ** test before it knows the cost to varint-encode the term and
4804: ** doclist lengths. At some point, change to
4805: ** pWriter->data.nData-iTermData>STANDALONE_MIN.
4806: */
4807: if( nTerm+nActualData>STANDALONE_MIN ){
4808: /* Push leaf node from before this term. */
4809: if( iTermData>0 ){
4810: rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
4811: if( rc!=SQLITE_OK ) return rc;
4812:
4813: pWriter->nTermDistinct = nTermDistinct;
4814: }
4815:
4816: /* Fix the encoded doclist length. */
4817: iDoclistData += n - nActual;
4818: memcpy(pWriter->data.pData+iDoclistData, c, nActual);
4819:
4820: /* Push the standalone leaf node. */
4821: rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
4822: if( rc!=SQLITE_OK ) return rc;
4823:
4824: /* Leave the node empty. */
4825: dataBufferReset(&pWriter->data);
4826:
4827: return rc;
4828: }
4829:
4830: /* At this point, we know that the doclist was small, so do the
4831: ** memmove if indicated.
4832: */
4833: if( nActual<n ){
4834: memmove(pWriter->data.pData+iDoclistData+nActual,
4835: pWriter->data.pData+iDoclistData+n,
4836: pWriter->data.nData-(iDoclistData+n));
4837: pWriter->data.nData -= n-nActual;
4838: }
4839:
4840: /* Replace written length with actual length. */
4841: memcpy(pWriter->data.pData+iDoclistData, c, nActual);
4842:
4843: /* If the node is too large, break things up. */
4844: /* TODO(shess) This test matches leafWriterStep(), which does this
4845: ** test before it knows the cost to varint-encode the term and
4846: ** doclist lengths. At some point, change to
4847: ** pWriter->data.nData>LEAF_MAX.
4848: */
4849: if( iTermData+nTerm+nActualData>LEAF_MAX ){
4850: /* Flush out the leading data as a node */
4851: rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
4852: if( rc!=SQLITE_OK ) return rc;
4853:
4854: pWriter->nTermDistinct = nTermDistinct;
4855:
4856: /* Rebuild header using the current term */
4857: n = putVarint(pWriter->data.pData, 0);
4858: n += putVarint(pWriter->data.pData+n, nTerm);
4859: memcpy(pWriter->data.pData+n, pTerm, nTerm);
4860: n += nTerm;
4861:
4862: /* There should always be room, because the previous encoding
4863: ** included all data necessary to construct the term.
4864: */
4865: assert( n<iDoclistData );
4866: /* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
4867: ** following memcpy() is safe (as opposed to needing a memmove).
4868: */
4869: assert( 2*STANDALONE_MIN<=LEAF_MAX );
4870: assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
4871: memcpy(pWriter->data.pData+n,
4872: pWriter->data.pData+iDoclistData,
4873: pWriter->data.nData-iDoclistData);
4874: pWriter->data.nData -= iDoclistData-n;
4875: }
4876: ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
4877:
4878: return SQLITE_OK;
4879: }
4880:
4881: /* Push pTerm[nTerm] along with the doclist data to the leaf layer of
4882: ** %_segments.
4883: */
4884: /* TODO(shess) Revise writeZeroSegment() so that doclists are
4885: ** constructed directly in pWriter->data.
4886: */
4887: static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter,
4888: const char *pTerm, int nTerm,
4889: const char *pData, int nData){
4890: int rc;
4891: DLReader reader;
4892:
4893: dlrInit(&reader, DL_DEFAULT, pData, nData);
4894: rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
4895: dlrDestroy(&reader);
4896:
4897: return rc;
4898: }
4899:
4900:
4901: /****************************************************************/
4902: /* LeafReader is used to iterate over an individual leaf node. */
4903: typedef struct LeafReader {
4904: DataBuffer term; /* copy of current term. */
4905:
4906: const char *pData; /* data for current term. */
4907: int nData;
4908: } LeafReader;
4909:
4910: static void leafReaderDestroy(LeafReader *pReader){
4911: dataBufferDestroy(&pReader->term);
4912: SCRAMBLE(pReader);
4913: }
4914:
4915: static int leafReaderAtEnd(LeafReader *pReader){
4916: return pReader->nData<=0;
4917: }
4918:
4919: /* Access the current term. */
4920: static int leafReaderTermBytes(LeafReader *pReader){
4921: return pReader->term.nData;
4922: }
4923: static const char *leafReaderTerm(LeafReader *pReader){
4924: assert( pReader->term.nData>0 );
4925: return pReader->term.pData;
4926: }
4927:
4928: /* Access the doclist data for the current term. */
4929: static int leafReaderDataBytes(LeafReader *pReader){
4930: int nData;
4931: assert( pReader->term.nData>0 );
4932: getVarint32(pReader->pData, &nData);
4933: return nData;
4934: }
4935: static const char *leafReaderData(LeafReader *pReader){
4936: int n, nData;
4937: assert( pReader->term.nData>0 );
4938: n = getVarint32(pReader->pData, &nData);
4939: return pReader->pData+n;
4940: }
4941:
4942: static void leafReaderInit(const char *pData, int nData,
4943: LeafReader *pReader){
4944: int nTerm, n;
4945:
4946: assert( nData>0 );
4947: assert( pData[0]=='\0' );
4948:
4949: CLEAR(pReader);
4950:
4951: /* Read the first term, skipping the header byte. */
4952: n = getVarint32(pData+1, &nTerm);
4953: dataBufferInit(&pReader->term, nTerm);
4954: dataBufferReplace(&pReader->term, pData+1+n, nTerm);
4955:
4956: /* Position after the first term. */
4957: assert( 1+n+nTerm<nData );
4958: pReader->pData = pData+1+n+nTerm;
4959: pReader->nData = nData-1-n-nTerm;
4960: }
4961:
4962: /* Step the reader forward to the next term. */
4963: static void leafReaderStep(LeafReader *pReader){
4964: int n, nData, nPrefix, nSuffix;
4965: assert( !leafReaderAtEnd(pReader) );
4966:
4967: /* Skip previous entry's data block. */
4968: n = getVarint32(pReader->pData, &nData);
4969: assert( n+nData<=pReader->nData );
4970: pReader->pData += n+nData;
4971: pReader->nData -= n+nData;
4972:
4973: if( !leafReaderAtEnd(pReader) ){
4974: /* Construct the new term using a prefix from the old term plus a
4975: ** suffix from the leaf data.
4976: */
4977: n = getVarint32(pReader->pData, &nPrefix);
4978: n += getVarint32(pReader->pData+n, &nSuffix);
4979: assert( n+nSuffix<pReader->nData );
4980: pReader->term.nData = nPrefix;
4981: dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
4982:
4983: pReader->pData += n+nSuffix;
4984: pReader->nData -= n+nSuffix;
4985: }
4986: }
4987:
4988: /* strcmp-style comparison of pReader's current term against pTerm.
4989: ** If isPrefix, equality means equal through nTerm bytes.
4990: */
4991: static int leafReaderTermCmp(LeafReader *pReader,
4992: const char *pTerm, int nTerm, int isPrefix){
4993: int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
4994: if( n==0 ){
4995: if( pReader->term.nData>0 ) return -1;
4996: if(nTerm>0 ) return 1;
4997: return 0;
4998: }
4999:
5000: c = memcmp(pReader->term.pData, pTerm, n);
5001: if( c!=0 ) return c;
5002: if( isPrefix && n==nTerm ) return 0;
5003: return pReader->term.nData - nTerm;
5004: }
5005:
5006:
5007: /****************************************************************/
5008: /* LeavesReader wraps LeafReader to allow iterating over the entire
5009: ** leaf layer of the tree.
5010: */
5011: typedef struct LeavesReader {
5012: int idx; /* Index within the segment. */
5013:
5014: sqlite3_stmt *pStmt; /* Statement we're streaming leaves from. */
5015: int eof; /* we've seen SQLITE_DONE from pStmt. */
5016:
5017: LeafReader leafReader; /* reader for the current leaf. */
5018: DataBuffer rootData; /* root data for inline. */
5019: } LeavesReader;
5020:
5021: /* Access the current term. */
5022: static int leavesReaderTermBytes(LeavesReader *pReader){
5023: assert( !pReader->eof );
5024: return leafReaderTermBytes(&pReader->leafReader);
5025: }
5026: static const char *leavesReaderTerm(LeavesReader *pReader){
5027: assert( !pReader->eof );
5028: return leafReaderTerm(&pReader->leafReader);
5029: }
5030:
5031: /* Access the doclist data for the current term. */
5032: static int leavesReaderDataBytes(LeavesReader *pReader){
5033: assert( !pReader->eof );
5034: return leafReaderDataBytes(&pReader->leafReader);
5035: }
5036: static const char *leavesReaderData(LeavesReader *pReader){
5037: assert( !pReader->eof );
5038: return leafReaderData(&pReader->leafReader);
5039: }
5040:
5041: static int leavesReaderAtEnd(LeavesReader *pReader){
5042: return pReader->eof;
5043: }
5044:
5045: /* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
5046: ** leaving the statement handle open, which locks the table.
5047: */
5048: /* TODO(shess) This "solution" is not satisfactory. Really, there
5049: ** should be check-in function for all statement handles which
5050: ** arranges to call sqlite3_reset(). This most likely will require
5051: ** modification to control flow all over the place, though, so for now
5052: ** just punt.
5053: **
5054: ** Note the the current system assumes that segment merges will run to
5055: ** completion, which is why this particular probably hasn't arisen in
5056: ** this case. Probably a brittle assumption.
5057: */
5058: static int leavesReaderReset(LeavesReader *pReader){
5059: return sqlite3_reset(pReader->pStmt);
5060: }
5061:
5062: static void leavesReaderDestroy(LeavesReader *pReader){
5063: /* If idx is -1, that means we're using a non-cached statement
5064: ** handle in the optimize() case, so we need to release it.
5065: */
5066: if( pReader->pStmt!=NULL && pReader->idx==-1 ){
5067: sqlite3_finalize(pReader->pStmt);
5068: }
5069: leafReaderDestroy(&pReader->leafReader);
5070: dataBufferDestroy(&pReader->rootData);
5071: SCRAMBLE(pReader);
5072: }
5073:
5074: /* Initialize pReader with the given root data (if iStartBlockid==0
5075: ** the leaf data was entirely contained in the root), or from the
5076: ** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
5077: */
5078: static int leavesReaderInit(fulltext_vtab *v,
5079: int idx,
5080: sqlite_int64 iStartBlockid,
5081: sqlite_int64 iEndBlockid,
5082: const char *pRootData, int nRootData,
5083: LeavesReader *pReader){
5084: CLEAR(pReader);
5085: pReader->idx = idx;
5086:
5087: dataBufferInit(&pReader->rootData, 0);
5088: if( iStartBlockid==0 ){
5089: /* Entire leaf level fit in root data. */
5090: dataBufferReplace(&pReader->rootData, pRootData, nRootData);
5091: leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
5092: &pReader->leafReader);
5093: }else{
5094: sqlite3_stmt *s;
5095: int rc = sql_get_leaf_statement(v, idx, &s);
5096: if( rc!=SQLITE_OK ) return rc;
5097:
5098: rc = sqlite3_bind_int64(s, 1, iStartBlockid);
5099: if( rc!=SQLITE_OK ) return rc;
5100:
5101: rc = sqlite3_bind_int64(s, 2, iEndBlockid);
5102: if( rc!=SQLITE_OK ) return rc;
5103:
5104: rc = sqlite3_step(s);
5105: if( rc==SQLITE_DONE ){
5106: pReader->eof = 1;
5107: return SQLITE_OK;
5108: }
5109: if( rc!=SQLITE_ROW ) return rc;
5110:
5111: pReader->pStmt = s;
5112: leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
5113: sqlite3_column_bytes(pReader->pStmt, 0),
5114: &pReader->leafReader);
5115: }
5116: return SQLITE_OK;
5117: }
5118:
5119: /* Step the current leaf forward to the next term. If we reach the
5120: ** end of the current leaf, step forward to the next leaf block.
5121: */
5122: static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
5123: assert( !leavesReaderAtEnd(pReader) );
5124: leafReaderStep(&pReader->leafReader);
5125:
5126: if( leafReaderAtEnd(&pReader->leafReader) ){
5127: int rc;
5128: if( pReader->rootData.pData ){
5129: pReader->eof = 1;
5130: return SQLITE_OK;
5131: }
5132: rc = sqlite3_step(pReader->pStmt);
5133: if( rc!=SQLITE_ROW ){
5134: pReader->eof = 1;
5135: return rc==SQLITE_DONE ? SQLITE_OK : rc;
5136: }
5137: leafReaderDestroy(&pReader->leafReader);
5138: leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
5139: sqlite3_column_bytes(pReader->pStmt, 0),
5140: &pReader->leafReader);
5141: }
5142: return SQLITE_OK;
5143: }
5144:
5145: /* Order LeavesReaders by their term, ignoring idx. Readers at eof
5146: ** always sort to the end.
5147: */
5148: static int leavesReaderTermCmp(LeavesReader *lr1, LeavesReader *lr2){
5149: if( leavesReaderAtEnd(lr1) ){
5150: if( leavesReaderAtEnd(lr2) ) return 0;
5151: return 1;
5152: }
5153: if( leavesReaderAtEnd(lr2) ) return -1;
5154:
5155: return leafReaderTermCmp(&lr1->leafReader,
5156: leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
5157: 0);
5158: }
5159:
5160: /* Similar to leavesReaderTermCmp(), with additional ordering by idx
5161: ** so that older segments sort before newer segments.
5162: */
5163: static int leavesReaderCmp(LeavesReader *lr1, LeavesReader *lr2){
5164: int c = leavesReaderTermCmp(lr1, lr2);
5165: if( c!=0 ) return c;
5166: return lr1->idx-lr2->idx;
5167: }
5168:
5169: /* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its
5170: ** sorted position.
5171: */
5172: static void leavesReaderReorder(LeavesReader *pLr, int nLr){
5173: while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
5174: LeavesReader tmp = pLr[0];
5175: pLr[0] = pLr[1];
5176: pLr[1] = tmp;
5177: nLr--;
5178: pLr++;
5179: }
5180: }
5181:
5182: /* Initializes pReaders with the segments from level iLevel, returning
5183: ** the number of segments in *piReaders. Leaves pReaders in sorted
5184: ** order.
5185: */
5186: static int leavesReadersInit(fulltext_vtab *v, int iLevel,
5187: LeavesReader *pReaders, int *piReaders){
5188: sqlite3_stmt *s;
5189: int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s);
5190: if( rc!=SQLITE_OK ) return rc;
5191:
5192: rc = sqlite3_bind_int(s, 1, iLevel);
5193: if( rc!=SQLITE_OK ) return rc;
5194:
5195: i = 0;
5196: while( (rc = sqlite3_step(s))==SQLITE_ROW ){
5197: sqlite_int64 iStart = sqlite3_column_int64(s, 0);
5198: sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
5199: const char *pRootData = sqlite3_column_blob(s, 2);
5200: int nRootData = sqlite3_column_bytes(s, 2);
5201:
5202: assert( i<MERGE_COUNT );
5203: rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
5204: &pReaders[i]);
5205: if( rc!=SQLITE_OK ) break;
5206:
5207: i++;
5208: }
5209: if( rc!=SQLITE_DONE ){
5210: while( i-->0 ){
5211: leavesReaderDestroy(&pReaders[i]);
5212: }
5213: return rc;
5214: }
5215:
5216: *piReaders = i;
5217:
5218: /* Leave our results sorted by term, then age. */
5219: while( i-- ){
5220: leavesReaderReorder(pReaders+i, *piReaders-i);
5221: }
5222: return SQLITE_OK;
5223: }
5224:
5225: /* Merge doclists from pReaders[nReaders] into a single doclist, which
5226: ** is written to pWriter. Assumes pReaders is ordered oldest to
5227: ** newest.
5228: */
5229: /* TODO(shess) Consider putting this inline in segmentMerge(). */
5230: static int leavesReadersMerge(fulltext_vtab *v,
5231: LeavesReader *pReaders, int nReaders,
5232: LeafWriter *pWriter){
5233: DLReader dlReaders[MERGE_COUNT];
5234: const char *pTerm = leavesReaderTerm(pReaders);
5235: int i, nTerm = leavesReaderTermBytes(pReaders);
5236:
5237: assert( nReaders<=MERGE_COUNT );
5238:
5239: for(i=0; i<nReaders; i++){
5240: dlrInit(&dlReaders[i], DL_DEFAULT,
5241: leavesReaderData(pReaders+i),
5242: leavesReaderDataBytes(pReaders+i));
5243: }
5244:
5245: return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
5246: }
5247:
5248: /* Forward ref due to mutual recursion with segdirNextIndex(). */
5249: static int segmentMerge(fulltext_vtab *v, int iLevel);
5250:
5251: /* Put the next available index at iLevel into *pidx. If iLevel
5252: ** already has MERGE_COUNT segments, they are merged to a higher
5253: ** level to make room.
5254: */
5255: static int segdirNextIndex(fulltext_vtab *v, int iLevel, int *pidx){
5256: int rc = segdir_max_index(v, iLevel, pidx);
5257: if( rc==SQLITE_DONE ){ /* No segments at iLevel. */
5258: *pidx = 0;
5259: }else if( rc==SQLITE_ROW ){
5260: if( *pidx==(MERGE_COUNT-1) ){
5261: rc = segmentMerge(v, iLevel);
5262: if( rc!=SQLITE_OK ) return rc;
5263: *pidx = 0;
5264: }else{
5265: (*pidx)++;
5266: }
5267: }else{
5268: return rc;
5269: }
5270: return SQLITE_OK;
5271: }
5272:
5273: /* Merge MERGE_COUNT segments at iLevel into a new segment at
5274: ** iLevel+1. If iLevel+1 is already full of segments, those will be
5275: ** merged to make room.
5276: */
5277: static int segmentMerge(fulltext_vtab *v, int iLevel){
5278: LeafWriter writer;
5279: LeavesReader lrs[MERGE_COUNT];
5280: int i, rc, idx = 0;
5281:
5282: /* Determine the next available segment index at the next level,
5283: ** merging as necessary.
5284: */
5285: rc = segdirNextIndex(v, iLevel+1, &idx);
5286: if( rc!=SQLITE_OK ) return rc;
5287:
5288: /* TODO(shess) This assumes that we'll always see exactly
5289: ** MERGE_COUNT segments to merge at a given level. That will be
5290: ** broken if we allow the developer to request preemptive or
5291: ** deferred merging.
5292: */
5293: memset(&lrs, '\0', sizeof(lrs));
5294: rc = leavesReadersInit(v, iLevel, lrs, &i);
5295: if( rc!=SQLITE_OK ) return rc;
5296: assert( i==MERGE_COUNT );
5297:
5298: leafWriterInit(iLevel+1, idx, &writer);
5299:
5300: /* Since leavesReaderReorder() pushes readers at eof to the end,
5301: ** when the first reader is empty, all will be empty.
5302: */
5303: while( !leavesReaderAtEnd(lrs) ){
5304: /* Figure out how many readers share their next term. */
5305: for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
5306: if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
5307: }
5308:
5309: rc = leavesReadersMerge(v, lrs, i, &writer);
5310: if( rc!=SQLITE_OK ) goto err;
5311:
5312: /* Step forward those that were merged. */
5313: while( i-->0 ){
5314: rc = leavesReaderStep(v, lrs+i);
5315: if( rc!=SQLITE_OK ) goto err;
5316:
5317: /* Reorder by term, then by age. */
5318: leavesReaderReorder(lrs+i, MERGE_COUNT-i);
5319: }
5320: }
5321:
5322: for(i=0; i<MERGE_COUNT; i++){
5323: leavesReaderDestroy(&lrs[i]);
5324: }
5325:
5326: rc = leafWriterFinalize(v, &writer);
5327: leafWriterDestroy(&writer);
5328: if( rc!=SQLITE_OK ) return rc;
5329:
5330: /* Delete the merged segment data. */
5331: return segdir_delete(v, iLevel);
5332:
5333: err:
5334: for(i=0; i<MERGE_COUNT; i++){
5335: leavesReaderDestroy(&lrs[i]);
5336: }
5337: leafWriterDestroy(&writer);
5338: return rc;
5339: }
5340:
5341: /* Accumulate the union of *acc and *pData into *acc. */
5342: static void docListAccumulateUnion(DataBuffer *acc,
5343: const char *pData, int nData) {
5344: DataBuffer tmp = *acc;
5345: dataBufferInit(acc, tmp.nData+nData);
5346: docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
5347: dataBufferDestroy(&tmp);
5348: }
5349:
5350: /* TODO(shess) It might be interesting to explore different merge
5351: ** strategies, here. For instance, since this is a sorted merge, we
5352: ** could easily merge many doclists in parallel. With some
5353: ** comprehension of the storage format, we could merge all of the
5354: ** doclists within a leaf node directly from the leaf node's storage.
5355: ** It may be worthwhile to merge smaller doclists before larger
5356: ** doclists, since they can be traversed more quickly - but the
5357: ** results may have less overlap, making them more expensive in a
5358: ** different way.
5359: */
5360:
5361: /* Scan pReader for pTerm/nTerm, and merge the term's doclist over
5362: ** *out (any doclists with duplicate docids overwrite those in *out).
5363: ** Internal function for loadSegmentLeaf().
5364: */
5365: static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
5366: const char *pTerm, int nTerm, int isPrefix,
5367: DataBuffer *out){
5368: /* doclist data is accumulated into pBuffers similar to how one does
5369: ** increment in binary arithmetic. If index 0 is empty, the data is
5370: ** stored there. If there is data there, it is merged and the
5371: ** results carried into position 1, with further merge-and-carry
5372: ** until an empty position is found.
5373: */
5374: DataBuffer *pBuffers = NULL;
5375: int nBuffers = 0, nMaxBuffers = 0, rc;
5376:
5377: assert( nTerm>0 );
5378:
5379: for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
5380: rc=leavesReaderStep(v, pReader)){
5381: /* TODO(shess) Really want leavesReaderTermCmp(), but that name is
5382: ** already taken to compare the terms of two LeavesReaders. Think
5383: ** on a better name. [Meanwhile, break encapsulation rather than
5384: ** use a confusing name.]
5385: */
5386: int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
5387: if( c>0 ) break; /* Past any possible matches. */
5388: if( c==0 ){
5389: const char *pData = leavesReaderData(pReader);
5390: int iBuffer, nData = leavesReaderDataBytes(pReader);
5391:
5392: /* Find the first empty buffer. */
5393: for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
5394: if( 0==pBuffers[iBuffer].nData ) break;
5395: }
5396:
5397: /* Out of buffers, add an empty one. */
5398: if( iBuffer==nBuffers ){
5399: if( nBuffers==nMaxBuffers ){
5400: DataBuffer *p;
5401: nMaxBuffers += 20;
5402:
5403: /* Manual realloc so we can handle NULL appropriately. */
5404: p = sqlite3_malloc(nMaxBuffers*sizeof(*pBuffers));
5405: if( p==NULL ){
5406: rc = SQLITE_NOMEM;
5407: break;
5408: }
5409:
5410: if( nBuffers>0 ){
5411: assert(pBuffers!=NULL);
5412: memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers));
5413: sqlite3_free(pBuffers);
5414: }
5415: pBuffers = p;
5416: }
5417: dataBufferInit(&(pBuffers[nBuffers]), 0);
5418: nBuffers++;
5419: }
5420:
5421: /* At this point, must have an empty at iBuffer. */
5422: assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
5423:
5424: /* If empty was first buffer, no need for merge logic. */
5425: if( iBuffer==0 ){
5426: dataBufferReplace(&(pBuffers[0]), pData, nData);
5427: }else{
5428: /* pAcc is the empty buffer the merged data will end up in. */
5429: DataBuffer *pAcc = &(pBuffers[iBuffer]);
5430: DataBuffer *p = &(pBuffers[0]);
5431:
5432: /* Handle position 0 specially to avoid need to prime pAcc
5433: ** with pData/nData.
5434: */
5435: dataBufferSwap(p, pAcc);
5436: docListAccumulateUnion(pAcc, pData, nData);
5437:
5438: /* Accumulate remaining doclists into pAcc. */
5439: for(++p; p<pAcc; ++p){
5440: docListAccumulateUnion(pAcc, p->pData, p->nData);
5441:
5442: /* dataBufferReset() could allow a large doclist to blow up
5443: ** our memory requirements.
5444: */
5445: if( p->nCapacity<1024 ){
5446: dataBufferReset(p);
5447: }else{
5448: dataBufferDestroy(p);
5449: dataBufferInit(p, 0);
5450: }
5451: }
5452: }
5453: }
5454: }
5455:
5456: /* Union all the doclists together into *out. */
5457: /* TODO(shess) What if *out is big? Sigh. */
5458: if( rc==SQLITE_OK && nBuffers>0 ){
5459: int iBuffer;
5460: for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
5461: if( pBuffers[iBuffer].nData>0 ){
5462: if( out->nData==0 ){
5463: dataBufferSwap(out, &(pBuffers[iBuffer]));
5464: }else{
5465: docListAccumulateUnion(out, pBuffers[iBuffer].pData,
5466: pBuffers[iBuffer].nData);
5467: }
5468: }
5469: }
5470: }
5471:
5472: while( nBuffers-- ){
5473: dataBufferDestroy(&(pBuffers[nBuffers]));
5474: }
5475: if( pBuffers!=NULL ) sqlite3_free(pBuffers);
5476:
5477: return rc;
5478: }
5479:
5480: /* Call loadSegmentLeavesInt() with pData/nData as input. */
5481: static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData,
5482: const char *pTerm, int nTerm, int isPrefix,
5483: DataBuffer *out){
5484: LeavesReader reader;
5485: int rc;
5486:
5487: assert( nData>1 );
5488: assert( *pData=='\0' );
5489: rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
5490: if( rc!=SQLITE_OK ) return rc;
5491:
5492: rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
5493: leavesReaderReset(&reader);
5494: leavesReaderDestroy(&reader);
5495: return rc;
5496: }
5497:
5498: /* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
5499: ** iEndLeaf (inclusive) as input, and merge the resulting doclist into
5500: ** out.
5501: */
5502: static int loadSegmentLeaves(fulltext_vtab *v,
5503: sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
5504: const char *pTerm, int nTerm, int isPrefix,
5505: DataBuffer *out){
5506: int rc;
5507: LeavesReader reader;
5508:
5509: assert( iStartLeaf<=iEndLeaf );
5510: rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
5511: if( rc!=SQLITE_OK ) return rc;
5512:
5513: rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
5514: leavesReaderReset(&reader);
5515: leavesReaderDestroy(&reader);
5516: return rc;
5517: }
5518:
5519: /* Taking pData/nData as an interior node, find the sequence of child
5520: ** nodes which could include pTerm/nTerm/isPrefix. Note that the
5521: ** interior node terms logically come between the blocks, so there is
5522: ** one more blockid than there are terms (that block contains terms >=
5523: ** the last interior-node term).
5524: */
5525: /* TODO(shess) The calling code may already know that the end child is
5526: ** not worth calculating, because the end may be in a later sibling
5527: ** node. Consider whether breaking symmetry is worthwhile. I suspect
5528: ** it is not worthwhile.
5529: */
5530: static void getChildrenContaining(const char *pData, int nData,
5531: const char *pTerm, int nTerm, int isPrefix,
5532: sqlite_int64 *piStartChild,
5533: sqlite_int64 *piEndChild){
5534: InteriorReader reader;
5535:
5536: assert( nData>1 );
5537: assert( *pData!='\0' );
5538: interiorReaderInit(pData, nData, &reader);
5539:
5540: /* Scan for the first child which could contain pTerm/nTerm. */
5541: while( !interiorReaderAtEnd(&reader) ){
5542: if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
5543: interiorReaderStep(&reader);
5544: }
5545: *piStartChild = interiorReaderCurrentBlockid(&reader);
5546:
5547: /* Keep scanning to find a term greater than our term, using prefix
5548: ** comparison if indicated. If isPrefix is false, this will be the
5549: ** same blockid as the starting block.
5550: */
5551: while( !interiorReaderAtEnd(&reader) ){
5552: if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
5553: interiorReaderStep(&reader);
5554: }
5555: *piEndChild = interiorReaderCurrentBlockid(&reader);
5556:
5557: interiorReaderDestroy(&reader);
5558:
5559: /* Children must ascend, and if !prefix, both must be the same. */
5560: assert( *piEndChild>=*piStartChild );
5561: assert( isPrefix || *piStartChild==*piEndChild );
5562: }
5563:
5564: /* Read block at iBlockid and pass it with other params to
5565: ** getChildrenContaining().
5566: */
5567: static int loadAndGetChildrenContaining(
5568: fulltext_vtab *v,
5569: sqlite_int64 iBlockid,
5570: const char *pTerm, int nTerm, int isPrefix,
5571: sqlite_int64 *piStartChild, sqlite_int64 *piEndChild
5572: ){
5573: sqlite3_stmt *s = NULL;
5574: int rc;
5575:
5576: assert( iBlockid!=0 );
5577: assert( pTerm!=NULL );
5578: assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */
5579: assert( piStartChild!=NULL );
5580: assert( piEndChild!=NULL );
5581:
5582: rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
5583: if( rc!=SQLITE_OK ) return rc;
5584:
5585: rc = sqlite3_bind_int64(s, 1, iBlockid);
5586: if( rc!=SQLITE_OK ) return rc;
5587:
5588: rc = sqlite3_step(s);
5589: if( rc==SQLITE_DONE ) return SQLITE_ERROR;
5590: if( rc!=SQLITE_ROW ) return rc;
5591:
5592: getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
5593: pTerm, nTerm, isPrefix, piStartChild, piEndChild);
5594:
5595: /* We expect only one row. We must execute another sqlite3_step()
5596: * to complete the iteration; otherwise the table will remain
5597: * locked. */
5598: rc = sqlite3_step(s);
5599: if( rc==SQLITE_ROW ) return SQLITE_ERROR;
5600: if( rc!=SQLITE_DONE ) return rc;
5601:
5602: return SQLITE_OK;
5603: }
5604:
5605: /* Traverse the tree represented by pData[nData] looking for
5606: ** pTerm[nTerm], placing its doclist into *out. This is internal to
5607: ** loadSegment() to make error-handling cleaner.
5608: */
5609: static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData,
5610: sqlite_int64 iLeavesEnd,
5611: const char *pTerm, int nTerm, int isPrefix,
5612: DataBuffer *out){
5613: /* Special case where root is a leaf. */
5614: if( *pData=='\0' ){
5615: return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
5616: }else{
5617: int rc;
5618: sqlite_int64 iStartChild, iEndChild;
5619:
5620: /* Process pData as an interior node, then loop down the tree
5621: ** until we find the set of leaf nodes to scan for the term.
5622: */
5623: getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
5624: &iStartChild, &iEndChild);
5625: while( iStartChild>iLeavesEnd ){
5626: sqlite_int64 iNextStart, iNextEnd;
5627: rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
5628: &iNextStart, &iNextEnd);
5629: if( rc!=SQLITE_OK ) return rc;
5630:
5631: /* If we've branched, follow the end branch, too. */
5632: if( iStartChild!=iEndChild ){
5633: sqlite_int64 iDummy;
5634: rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
5635: &iDummy, &iNextEnd);
5636: if( rc!=SQLITE_OK ) return rc;
5637: }
5638:
5639: assert( iNextStart<=iNextEnd );
5640: iStartChild = iNextStart;
5641: iEndChild = iNextEnd;
5642: }
5643: assert( iStartChild<=iLeavesEnd );
5644: assert( iEndChild<=iLeavesEnd );
5645:
5646: /* Scan through the leaf segments for doclists. */
5647: return loadSegmentLeaves(v, iStartChild, iEndChild,
5648: pTerm, nTerm, isPrefix, out);
5649: }
5650: }
5651:
5652: /* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
5653: ** merge its doclist over *out (any duplicate doclists read from the
5654: ** segment rooted at pData will overwrite those in *out).
5655: */
5656: /* TODO(shess) Consider changing this to determine the depth of the
5657: ** leaves using either the first characters of interior nodes (when
5658: ** ==1, we're one level above the leaves), or the first character of
5659: ** the root (which will describe the height of the tree directly).
5660: ** Either feels somewhat tricky to me.
5661: */
5662: /* TODO(shess) The current merge is likely to be slow for large
5663: ** doclists (though it should process from newest/smallest to
5664: ** oldest/largest, so it may not be that bad). It might be useful to
5665: ** modify things to allow for N-way merging. This could either be
5666: ** within a segment, with pairwise merges across segments, or across
5667: ** all segments at once.
5668: */
5669: static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
5670: sqlite_int64 iLeavesEnd,
5671: const char *pTerm, int nTerm, int isPrefix,
5672: DataBuffer *out){
5673: DataBuffer result;
5674: int rc;
5675:
5676: assert( nData>1 );
5677:
5678: /* This code should never be called with buffered updates. */
5679: assert( v->nPendingData<0 );
5680:
5681: dataBufferInit(&result, 0);
5682: rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
5683: pTerm, nTerm, isPrefix, &result);
5684: if( rc==SQLITE_OK && result.nData>0 ){
5685: if( out->nData==0 ){
5686: DataBuffer tmp = *out;
5687: *out = result;
5688: result = tmp;
5689: }else{
5690: DataBuffer merged;
5691: DLReader readers[2];
5692:
5693: dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
5694: dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
5695: dataBufferInit(&merged, out->nData+result.nData);
5696: docListMerge(&merged, readers, 2);
5697: dataBufferDestroy(out);
5698: *out = merged;
5699: dlrDestroy(&readers[0]);
5700: dlrDestroy(&readers[1]);
5701: }
5702: }
5703: dataBufferDestroy(&result);
5704: return rc;
5705: }
5706:
5707: /* Scan the database and merge together the posting lists for the term
5708: ** into *out.
5709: */
5710: static int termSelect(fulltext_vtab *v, int iColumn,
5711: const char *pTerm, int nTerm, int isPrefix,
5712: DocListType iType, DataBuffer *out){
5713: DataBuffer doclist;
5714: sqlite3_stmt *s;
5715: int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
5716: if( rc!=SQLITE_OK ) return rc;
5717:
5718: /* This code should never be called with buffered updates. */
5719: assert( v->nPendingData<0 );
5720:
5721: dataBufferInit(&doclist, 0);
5722:
5723: /* Traverse the segments from oldest to newest so that newer doclist
5724: ** elements for given docids overwrite older elements.
5725: */
5726: while( (rc = sqlite3_step(s))==SQLITE_ROW ){
5727: const char *pData = sqlite3_column_blob(s, 2);
5728: const int nData = sqlite3_column_bytes(s, 2);
5729: const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
5730: rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
5731: &doclist);
5732: if( rc!=SQLITE_OK ) goto err;
5733: }
5734: if( rc==SQLITE_DONE ){
5735: if( doclist.nData!=0 ){
5736: /* TODO(shess) The old term_select_all() code applied the column
5737: ** restrict as we merged segments, leading to smaller buffers.
5738: ** This is probably worthwhile to bring back, once the new storage
5739: ** system is checked in.
5740: */
5741: if( iColumn==v->nColumn) iColumn = -1;
5742: docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
5743: iColumn, iType, out);
5744: }
5745: rc = SQLITE_OK;
5746: }
5747:
5748: err:
5749: dataBufferDestroy(&doclist);
5750: return rc;
5751: }
5752:
5753: /****************************************************************/
5754: /* Used to hold hashtable data for sorting. */
5755: typedef struct TermData {
5756: const char *pTerm;
5757: int nTerm;
5758: DLCollector *pCollector;
5759: } TermData;
5760:
5761: /* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
5762: ** for equal, >0 for greater-than).
5763: */
5764: static int termDataCmp(const void *av, const void *bv){
5765: const TermData *a = (const TermData *)av;
5766: const TermData *b = (const TermData *)bv;
5767: int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
5768: int c = memcmp(a->pTerm, b->pTerm, n);
5769: if( c!=0 ) return c;
5770: return a->nTerm-b->nTerm;
5771: }
5772:
5773: /* Order pTerms data by term, then write a new level 0 segment using
5774: ** LeafWriter.
5775: */
5776: static int writeZeroSegment(fulltext_vtab *v, fts2Hash *pTerms){
5777: fts2HashElem *e;
5778: int idx, rc, i, n;
5779: TermData *pData;
5780: LeafWriter writer;
5781: DataBuffer dl;
5782:
5783: /* Determine the next index at level 0, merging as necessary. */
5784: rc = segdirNextIndex(v, 0, &idx);
5785: if( rc!=SQLITE_OK ) return rc;
5786:
5787: n = fts2HashCount(pTerms);
5788: pData = sqlite3_malloc(n*sizeof(TermData));
5789:
5790: for(i = 0, e = fts2HashFirst(pTerms); e; i++, e = fts2HashNext(e)){
5791: assert( i<n );
5792: pData[i].pTerm = fts2HashKey(e);
5793: pData[i].nTerm = fts2HashKeysize(e);
5794: pData[i].pCollector = fts2HashData(e);
5795: }
5796: assert( i==n );
5797:
5798: /* TODO(shess) Should we allow user-defined collation sequences,
5799: ** here? I think we only need that once we support prefix searches.
5800: */
5801: if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
5802:
5803: /* TODO(shess) Refactor so that we can write directly to the segment
5804: ** DataBuffer, as happens for segment merges.
5805: */
5806: leafWriterInit(0, idx, &writer);
5807: dataBufferInit(&dl, 0);
5808: for(i=0; i<n; i++){
5809: dataBufferReset(&dl);
5810: dlcAddDoclist(pData[i].pCollector, &dl);
5811: rc = leafWriterStep(v, &writer,
5812: pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
5813: if( rc!=SQLITE_OK ) goto err;
5814: }
5815: rc = leafWriterFinalize(v, &writer);
5816:
5817: err:
5818: dataBufferDestroy(&dl);
5819: sqlite3_free(pData);
5820: leafWriterDestroy(&writer);
5821: return rc;
5822: }
5823:
5824: /* If pendingTerms has data, free it. */
5825: static int clearPendingTerms(fulltext_vtab *v){
5826: if( v->nPendingData>=0 ){
5827: fts2HashElem *e;
5828: for(e=fts2HashFirst(&v->pendingTerms); e; e=fts2HashNext(e)){
5829: dlcDelete(fts2HashData(e));
5830: }
5831: fts2HashClear(&v->pendingTerms);
5832: v->nPendingData = -1;
5833: }
5834: return SQLITE_OK;
5835: }
5836:
5837: /* If pendingTerms has data, flush it to a level-zero segment, and
5838: ** free it.
5839: */
5840: static int flushPendingTerms(fulltext_vtab *v){
5841: if( v->nPendingData>=0 ){
5842: int rc = writeZeroSegment(v, &v->pendingTerms);
5843: if( rc==SQLITE_OK ) clearPendingTerms(v);
5844: return rc;
5845: }
5846: return SQLITE_OK;
5847: }
5848:
5849: /* If pendingTerms is "too big", or docid is out of order, flush it.
5850: ** Regardless, be certain that pendingTerms is initialized for use.
5851: */
5852: static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
5853: /* TODO(shess) Explore whether partially flushing the buffer on
5854: ** forced-flush would provide better performance. I suspect that if
5855: ** we ordered the doclists by size and flushed the largest until the
5856: ** buffer was half empty, that would let the less frequent terms
5857: ** generate longer doclists.
5858: */
5859: if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){
5860: int rc = flushPendingTerms(v);
5861: if( rc!=SQLITE_OK ) return rc;
5862: }
5863: if( v->nPendingData<0 ){
5864: fts2HashInit(&v->pendingTerms, FTS2_HASH_STRING, 1);
5865: v->nPendingData = 0;
5866: }
5867: v->iPrevDocid = iDocid;
5868: return SQLITE_OK;
5869: }
5870:
5871: /* This function implements the xUpdate callback; it is the top-level entry
5872: * point for inserting, deleting or updating a row in a full-text table. */
5873: static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
5874: sqlite_int64 *pRowid){
5875: fulltext_vtab *v = (fulltext_vtab *) pVtab;
5876: int rc;
5877:
5878: TRACE(("FTS2 Update %p\n", pVtab));
5879:
5880: if( nArg<2 ){
5881: rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
5882: if( rc==SQLITE_OK ){
5883: /* If we just deleted the last row in the table, clear out the
5884: ** index data.
5885: */
5886: rc = content_exists(v);
5887: if( rc==SQLITE_ROW ){
5888: rc = SQLITE_OK;
5889: }else if( rc==SQLITE_DONE ){
5890: /* Clear the pending terms so we don't flush a useless level-0
5891: ** segment when the transaction closes.
5892: */
5893: rc = clearPendingTerms(v);
5894: if( rc==SQLITE_OK ){
5895: rc = segdir_delete_all(v);
5896: }
5897: }
5898: }
5899: } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
5900: /* An update:
5901: * ppArg[0] = old rowid
5902: * ppArg[1] = new rowid
5903: * ppArg[2..2+v->nColumn-1] = values
5904: * ppArg[2+v->nColumn] = value for magic column (we ignore this)
5905: */
5906: sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
5907: if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER ||
5908: sqlite3_value_int64(ppArg[1]) != rowid ){
5909: rc = SQLITE_ERROR; /* we don't allow changing the rowid */
5910: } else {
5911: assert( nArg==2+v->nColumn+1);
5912: rc = index_update(v, rowid, &ppArg[2]);
5913: }
5914: } else {
5915: /* An insert:
5916: * ppArg[1] = requested rowid
5917: * ppArg[2..2+v->nColumn-1] = values
5918: * ppArg[2+v->nColumn] = value for magic column (we ignore this)
5919: */
5920: assert( nArg==2+v->nColumn+1);
5921: rc = index_insert(v, ppArg[1], &ppArg[2], pRowid);
5922: }
5923:
5924: return rc;
5925: }
5926:
5927: static int fulltextSync(sqlite3_vtab *pVtab){
5928: TRACE(("FTS2 xSync()\n"));
5929: return flushPendingTerms((fulltext_vtab *)pVtab);
5930: }
5931:
5932: static int fulltextBegin(sqlite3_vtab *pVtab){
5933: fulltext_vtab *v = (fulltext_vtab *) pVtab;
5934: TRACE(("FTS2 xBegin()\n"));
5935:
5936: /* Any buffered updates should have been cleared by the previous
5937: ** transaction.
5938: */
5939: assert( v->nPendingData<0 );
5940: return clearPendingTerms(v);
5941: }
5942:
5943: static int fulltextCommit(sqlite3_vtab *pVtab){
5944: fulltext_vtab *v = (fulltext_vtab *) pVtab;
5945: TRACE(("FTS2 xCommit()\n"));
5946:
5947: /* Buffered updates should have been cleared by fulltextSync(). */
5948: assert( v->nPendingData<0 );
5949: return clearPendingTerms(v);
5950: }
5951:
5952: static int fulltextRollback(sqlite3_vtab *pVtab){
5953: TRACE(("FTS2 xRollback()\n"));
5954: return clearPendingTerms((fulltext_vtab *)pVtab);
5955: }
5956:
5957: /*
5958: ** Implementation of the snippet() function for FTS2
5959: */
5960: static void snippetFunc(
5961: sqlite3_context *pContext,
5962: int argc,
5963: sqlite3_value **argv
5964: ){
5965: fulltext_cursor *pCursor;
5966: if( argc<1 ) return;
5967: if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
5968: sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
5969: sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
5970: }else{
5971: const char *zStart = "<b>";
5972: const char *zEnd = "</b>";
5973: const char *zEllipsis = "<b>...</b>";
5974: memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
5975: if( argc>=2 ){
5976: zStart = (const char*)sqlite3_value_text(argv[1]);
5977: if( argc>=3 ){
5978: zEnd = (const char*)sqlite3_value_text(argv[2]);
5979: if( argc>=4 ){
5980: zEllipsis = (const char*)sqlite3_value_text(argv[3]);
5981: }
5982: }
5983: }
5984: snippetAllOffsets(pCursor);
5985: snippetText(pCursor, zStart, zEnd, zEllipsis);
5986: sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
5987: pCursor->snippet.nSnippet, SQLITE_STATIC);
5988: }
5989: }
5990:
5991: /*
5992: ** Implementation of the offsets() function for FTS2
5993: */
5994: static void snippetOffsetsFunc(
5995: sqlite3_context *pContext,
5996: int argc,
5997: sqlite3_value **argv
5998: ){
5999: fulltext_cursor *pCursor;
6000: if( argc<1 ) return;
6001: if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
6002: sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
6003: sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
6004: }else{
6005: memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
6006: snippetAllOffsets(pCursor);
6007: snippetOffsetText(&pCursor->snippet);
6008: sqlite3_result_text(pContext,
6009: pCursor->snippet.zOffset, pCursor->snippet.nOffset,
6010: SQLITE_STATIC);
6011: }
6012: }
6013:
6014: /* OptLeavesReader is nearly identical to LeavesReader, except that
6015: ** where LeavesReader is geared towards the merging of complete
6016: ** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader
6017: ** is geared towards implementation of the optimize() function, and
6018: ** can merge all segments simultaneously. This version may be
6019: ** somewhat less efficient than LeavesReader because it merges into an
6020: ** accumulator rather than doing an N-way merge, but since segment
6021: ** size grows exponentially (so segment count logrithmically) this is
6022: ** probably not an immediate problem.
6023: */
6024: /* TODO(shess): Prove that assertion, or extend the merge code to
6025: ** merge tree fashion (like the prefix-searching code does).
6026: */
6027: /* TODO(shess): OptLeavesReader and LeavesReader could probably be
6028: ** merged with little or no loss of performance for LeavesReader. The
6029: ** merged code would need to handle >MERGE_COUNT segments, and would
6030: ** also need to be able to optionally optimize away deletes.
6031: */
6032: typedef struct OptLeavesReader {
6033: /* Segment number, to order readers by age. */
6034: int segment;
6035: LeavesReader reader;
6036: } OptLeavesReader;
6037:
6038: static int optLeavesReaderAtEnd(OptLeavesReader *pReader){
6039: return leavesReaderAtEnd(&pReader->reader);
6040: }
6041: static int optLeavesReaderTermBytes(OptLeavesReader *pReader){
6042: return leavesReaderTermBytes(&pReader->reader);
6043: }
6044: static const char *optLeavesReaderData(OptLeavesReader *pReader){
6045: return leavesReaderData(&pReader->reader);
6046: }
6047: static int optLeavesReaderDataBytes(OptLeavesReader *pReader){
6048: return leavesReaderDataBytes(&pReader->reader);
6049: }
6050: static const char *optLeavesReaderTerm(OptLeavesReader *pReader){
6051: return leavesReaderTerm(&pReader->reader);
6052: }
6053: static int optLeavesReaderStep(fulltext_vtab *v, OptLeavesReader *pReader){
6054: return leavesReaderStep(v, &pReader->reader);
6055: }
6056: static int optLeavesReaderTermCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
6057: return leavesReaderTermCmp(&lr1->reader, &lr2->reader);
6058: }
6059: /* Order by term ascending, segment ascending (oldest to newest), with
6060: ** exhausted readers to the end.
6061: */
6062: static int optLeavesReaderCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
6063: int c = optLeavesReaderTermCmp(lr1, lr2);
6064: if( c!=0 ) return c;
6065: return lr1->segment-lr2->segment;
6066: }
6067: /* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that
6068: ** pLr[1..nLr-1] is already sorted.
6069: */
6070: static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){
6071: while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){
6072: OptLeavesReader tmp = pLr[0];
6073: pLr[0] = pLr[1];
6074: pLr[1] = tmp;
6075: nLr--;
6076: pLr++;
6077: }
6078: }
6079:
6080: /* optimize() helper function. Put the readers in order and iterate
6081: ** through them, merging doclists for matching terms into pWriter.
6082: ** Returns SQLITE_OK on success, or the SQLite error code which
6083: ** prevented success.
6084: */
6085: static int optimizeInternal(fulltext_vtab *v,
6086: OptLeavesReader *readers, int nReaders,
6087: LeafWriter *pWriter){
6088: int i, rc = SQLITE_OK;
6089: DataBuffer doclist, merged, tmp;
6090:
6091: /* Order the readers. */
6092: i = nReaders;
6093: while( i-- > 0 ){
6094: optLeavesReaderReorder(&readers[i], nReaders-i);
6095: }
6096:
6097: dataBufferInit(&doclist, LEAF_MAX);
6098: dataBufferInit(&merged, LEAF_MAX);
6099:
6100: /* Exhausted readers bubble to the end, so when the first reader is
6101: ** at eof, all are at eof.
6102: */
6103: while( !optLeavesReaderAtEnd(&readers[0]) ){
6104:
6105: /* Figure out how many readers share the next term. */
6106: for(i=1; i<nReaders && !optLeavesReaderAtEnd(&readers[i]); i++){
6107: if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
6108: }
6109:
6110: /* Special-case for no merge. */
6111: if( i==1 ){
6112: /* Trim deletions from the doclist. */
6113: dataBufferReset(&merged);
6114: docListTrim(DL_DEFAULT,
6115: optLeavesReaderData(&readers[0]),
6116: optLeavesReaderDataBytes(&readers[0]),
6117: -1, DL_DEFAULT, &merged);
6118: }else{
6119: DLReader dlReaders[MERGE_COUNT];
6120: int iReader, nReaders;
6121:
6122: /* Prime the pipeline with the first reader's doclist. After
6123: ** one pass index 0 will reference the accumulated doclist.
6124: */
6125: dlrInit(&dlReaders[0], DL_DEFAULT,
6126: optLeavesReaderData(&readers[0]),
6127: optLeavesReaderDataBytes(&readers[0]));
6128: iReader = 1;
6129:
6130: assert( iReader<i ); /* Must execute the loop at least once. */
6131: while( iReader<i ){
6132: /* Merge 16 inputs per pass. */
6133: for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
6134: iReader++, nReaders++ ){
6135: dlrInit(&dlReaders[nReaders], DL_DEFAULT,
6136: optLeavesReaderData(&readers[iReader]),
6137: optLeavesReaderDataBytes(&readers[iReader]));
6138: }
6139:
6140: /* Merge doclists and swap result into accumulator. */
6141: dataBufferReset(&merged);
6142: docListMerge(&merged, dlReaders, nReaders);
6143: tmp = merged;
6144: merged = doclist;
6145: doclist = tmp;
6146:
6147: while( nReaders-- > 0 ){
6148: dlrDestroy(&dlReaders[nReaders]);
6149: }
6150:
6151: /* Accumulated doclist to reader 0 for next pass. */
6152: dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
6153: }
6154:
6155: /* Destroy reader that was left in the pipeline. */
6156: dlrDestroy(&dlReaders[0]);
6157:
6158: /* Trim deletions from the doclist. */
6159: dataBufferReset(&merged);
6160: docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
6161: -1, DL_DEFAULT, &merged);
6162: }
6163:
6164: /* Only pass doclists with hits (skip if all hits deleted). */
6165: if( merged.nData>0 ){
6166: rc = leafWriterStep(v, pWriter,
6167: optLeavesReaderTerm(&readers[0]),
6168: optLeavesReaderTermBytes(&readers[0]),
6169: merged.pData, merged.nData);
6170: if( rc!=SQLITE_OK ) goto err;
6171: }
6172:
6173: /* Step merged readers to next term and reorder. */
6174: while( i-- > 0 ){
6175: rc = optLeavesReaderStep(v, &readers[i]);
6176: if( rc!=SQLITE_OK ) goto err;
6177:
6178: optLeavesReaderReorder(&readers[i], nReaders-i);
6179: }
6180: }
6181:
6182: err:
6183: dataBufferDestroy(&doclist);
6184: dataBufferDestroy(&merged);
6185: return rc;
6186: }
6187:
6188: /* Implement optimize() function for FTS3. optimize(t) merges all
6189: ** segments in the fts index into a single segment. 't' is the magic
6190: ** table-named column.
6191: */
6192: static void optimizeFunc(sqlite3_context *pContext,
6193: int argc, sqlite3_value **argv){
6194: fulltext_cursor *pCursor;
6195: if( argc>1 ){
6196: sqlite3_result_error(pContext, "excess arguments to optimize()",-1);
6197: }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
6198: sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
6199: sqlite3_result_error(pContext, "illegal first argument to optimize",-1);
6200: }else{
6201: fulltext_vtab *v;
6202: int i, rc, iMaxLevel;
6203: OptLeavesReader *readers;
6204: int nReaders;
6205: LeafWriter writer;
6206: sqlite3_stmt *s;
6207:
6208: memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
6209: v = cursor_vtab(pCursor);
6210:
6211: /* Flush any buffered updates before optimizing. */
6212: rc = flushPendingTerms(v);
6213: if( rc!=SQLITE_OK ) goto err;
6214:
6215: rc = segdir_count(v, &nReaders, &iMaxLevel);
6216: if( rc!=SQLITE_OK ) goto err;
6217: if( nReaders==0 || nReaders==1 ){
6218: sqlite3_result_text(pContext, "Index already optimal", -1,
6219: SQLITE_STATIC);
6220: return;
6221: }
6222:
6223: rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
6224: if( rc!=SQLITE_OK ) goto err;
6225:
6226: readers = sqlite3_malloc(nReaders*sizeof(readers[0]));
6227: if( readers==NULL ) goto err;
6228:
6229: /* Note that there will already be a segment at this position
6230: ** until we call segdir_delete() on iMaxLevel.
6231: */
6232: leafWriterInit(iMaxLevel, 0, &writer);
6233:
6234: i = 0;
6235: while( (rc = sqlite3_step(s))==SQLITE_ROW ){
6236: sqlite_int64 iStart = sqlite3_column_int64(s, 0);
6237: sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
6238: const char *pRootData = sqlite3_column_blob(s, 2);
6239: int nRootData = sqlite3_column_bytes(s, 2);
6240:
6241: assert( i<nReaders );
6242: rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
6243: &readers[i].reader);
6244: if( rc!=SQLITE_OK ) break;
6245:
6246: readers[i].segment = i;
6247: i++;
6248: }
6249:
6250: /* If we managed to successfully read them all, optimize them. */
6251: if( rc==SQLITE_DONE ){
6252: assert( i==nReaders );
6253: rc = optimizeInternal(v, readers, nReaders, &writer);
6254: }
6255:
6256: while( i-- > 0 ){
6257: leavesReaderDestroy(&readers[i].reader);
6258: }
6259: sqlite3_free(readers);
6260:
6261: /* If we've successfully gotten to here, delete the old segments
6262: ** and flush the interior structure of the new segment.
6263: */
6264: if( rc==SQLITE_OK ){
6265: for( i=0; i<=iMaxLevel; i++ ){
6266: rc = segdir_delete(v, i);
6267: if( rc!=SQLITE_OK ) break;
6268: }
6269:
6270: if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer);
6271: }
6272:
6273: leafWriterDestroy(&writer);
6274:
6275: if( rc!=SQLITE_OK ) goto err;
6276:
6277: sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
6278: return;
6279:
6280: /* TODO(shess): Error-handling needs to be improved along the
6281: ** lines of the dump_ functions.
6282: */
6283: err:
6284: {
6285: char buf[512];
6286: sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s",
6287: sqlite3_errmsg(sqlite3_context_db_handle(pContext)));
6288: sqlite3_result_error(pContext, buf, -1);
6289: }
6290: }
6291: }
6292:
6293: #ifdef SQLITE_TEST
6294: /* Generate an error of the form "<prefix>: <msg>". If msg is NULL,
6295: ** pull the error from the context's db handle.
6296: */
6297: static void generateError(sqlite3_context *pContext,
6298: const char *prefix, const char *msg){
6299: char buf[512];
6300: if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext));
6301: sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg);
6302: sqlite3_result_error(pContext, buf, -1);
6303: }
6304:
6305: /* Helper function to collect the set of terms in the segment into
6306: ** pTerms. The segment is defined by the leaf nodes between
6307: ** iStartBlockid and iEndBlockid, inclusive, or by the contents of
6308: ** pRootData if iStartBlockid is 0 (in which case the entire segment
6309: ** fit in a leaf).
6310: */
6311: static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s,
6312: fts2Hash *pTerms){
6313: const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0);
6314: const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
6315: const char *pRootData = sqlite3_column_blob(s, 2);
6316: const int nRootData = sqlite3_column_bytes(s, 2);
6317: LeavesReader reader;
6318: int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
6319: pRootData, nRootData, &reader);
6320: if( rc!=SQLITE_OK ) return rc;
6321:
6322: while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
6323: const char *pTerm = leavesReaderTerm(&reader);
6324: const int nTerm = leavesReaderTermBytes(&reader);
6325: void *oldValue = sqlite3Fts2HashFind(pTerms, pTerm, nTerm);
6326: void *newValue = (void *)((char *)oldValue+1);
6327:
6328: /* From the comment before sqlite3Fts2HashInsert in fts2_hash.c,
6329: ** the data value passed is returned in case of malloc failure.
6330: */
6331: if( newValue==sqlite3Fts2HashInsert(pTerms, pTerm, nTerm, newValue) ){
6332: rc = SQLITE_NOMEM;
6333: }else{
6334: rc = leavesReaderStep(v, &reader);
6335: }
6336: }
6337:
6338: leavesReaderDestroy(&reader);
6339: return rc;
6340: }
6341:
6342: /* Helper function to build the result string for dump_terms(). */
6343: static int generateTermsResult(sqlite3_context *pContext, fts2Hash *pTerms){
6344: int iTerm, nTerms, nResultBytes, iByte;
6345: char *result;
6346: TermData *pData;
6347: fts2HashElem *e;
6348:
6349: /* Iterate pTerms to generate an array of terms in pData for
6350: ** sorting.
6351: */
6352: nTerms = fts2HashCount(pTerms);
6353: assert( nTerms>0 );
6354: pData = sqlite3_malloc(nTerms*sizeof(TermData));
6355: if( pData==NULL ) return SQLITE_NOMEM;
6356:
6357: nResultBytes = 0;
6358: for(iTerm = 0, e = fts2HashFirst(pTerms); e; iTerm++, e = fts2HashNext(e)){
6359: nResultBytes += fts2HashKeysize(e)+1; /* Term plus trailing space */
6360: assert( iTerm<nTerms );
6361: pData[iTerm].pTerm = fts2HashKey(e);
6362: pData[iTerm].nTerm = fts2HashKeysize(e);
6363: pData[iTerm].pCollector = fts2HashData(e); /* unused */
6364: }
6365: assert( iTerm==nTerms );
6366:
6367: assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */
6368: result = sqlite3_malloc(nResultBytes);
6369: if( result==NULL ){
6370: sqlite3_free(pData);
6371: return SQLITE_NOMEM;
6372: }
6373:
6374: if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp);
6375:
6376: /* Read the terms in order to build the result. */
6377: iByte = 0;
6378: for(iTerm=0; iTerm<nTerms; ++iTerm){
6379: memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm);
6380: iByte += pData[iTerm].nTerm;
6381: result[iByte++] = ' ';
6382: }
6383: assert( iByte==nResultBytes );
6384: assert( result[nResultBytes-1]==' ' );
6385: result[nResultBytes-1] = '\0';
6386:
6387: /* Passes away ownership of result. */
6388: sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free);
6389: sqlite3_free(pData);
6390: return SQLITE_OK;
6391: }
6392:
6393: /* Implements dump_terms() for use in inspecting the fts2 index from
6394: ** tests. TEXT result containing the ordered list of terms joined by
6395: ** spaces. dump_terms(t, level, idx) dumps the terms for the segment
6396: ** specified by level, idx (in %_segdir), while dump_terms(t) dumps
6397: ** all terms in the index. In both cases t is the fts table's magic
6398: ** table-named column.
6399: */
6400: static void dumpTermsFunc(
6401: sqlite3_context *pContext,
6402: int argc, sqlite3_value **argv
6403: ){
6404: fulltext_cursor *pCursor;
6405: if( argc!=3 && argc!=1 ){
6406: generateError(pContext, "dump_terms", "incorrect arguments");
6407: }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
6408: sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
6409: generateError(pContext, "dump_terms", "illegal first argument");
6410: }else{
6411: fulltext_vtab *v;
6412: fts2Hash terms;
6413: sqlite3_stmt *s = NULL;
6414: int rc;
6415:
6416: memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
6417: v = cursor_vtab(pCursor);
6418:
6419: /* If passed only the cursor column, get all segments. Otherwise
6420: ** get the segment described by the following two arguments.
6421: */
6422: if( argc==1 ){
6423: rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
6424: }else{
6425: rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
6426: if( rc==SQLITE_OK ){
6427: rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1]));
6428: if( rc==SQLITE_OK ){
6429: rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2]));
6430: }
6431: }
6432: }
6433:
6434: if( rc!=SQLITE_OK ){
6435: generateError(pContext, "dump_terms", NULL);
6436: return;
6437: }
6438:
6439: /* Collect the terms for each segment. */
6440: sqlite3Fts2HashInit(&terms, FTS2_HASH_STRING, 1);
6441: while( (rc = sqlite3_step(s))==SQLITE_ROW ){
6442: rc = collectSegmentTerms(v, s, &terms);
6443: if( rc!=SQLITE_OK ) break;
6444: }
6445:
6446: if( rc!=SQLITE_DONE ){
6447: sqlite3_reset(s);
6448: generateError(pContext, "dump_terms", NULL);
6449: }else{
6450: const int nTerms = fts2HashCount(&terms);
6451: if( nTerms>0 ){
6452: rc = generateTermsResult(pContext, &terms);
6453: if( rc==SQLITE_NOMEM ){
6454: generateError(pContext, "dump_terms", "out of memory");
6455: }else{
6456: assert( rc==SQLITE_OK );
6457: }
6458: }else if( argc==3 ){
6459: /* The specific segment asked for could not be found. */
6460: generateError(pContext, "dump_terms", "segment not found");
6461: }else{
6462: /* No segments found. */
6463: /* TODO(shess): It should be impossible to reach this. This
6464: ** case can only happen for an empty table, in which case
6465: ** SQLite has no rows to call this function on.
6466: */
6467: sqlite3_result_null(pContext);
6468: }
6469: }
6470: sqlite3Fts2HashClear(&terms);
6471: }
6472: }
6473:
6474: /* Expand the DL_DEFAULT doclist in pData into a text result in
6475: ** pContext.
6476: */
6477: static void createDoclistResult(sqlite3_context *pContext,
6478: const char *pData, int nData){
6479: DataBuffer dump;
6480: DLReader dlReader;
6481:
6482: assert( pData!=NULL && nData>0 );
6483:
6484: dataBufferInit(&dump, 0);
6485: dlrInit(&dlReader, DL_DEFAULT, pData, nData);
6486: for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
6487: char buf[256];
6488: PLReader plReader;
6489:
6490: plrInit(&plReader, &dlReader);
6491: if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){
6492: sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
6493: dataBufferAppend(&dump, buf, strlen(buf));
6494: }else{
6495: int iColumn = plrColumn(&plReader);
6496:
6497: sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[",
6498: dlrDocid(&dlReader), iColumn);
6499: dataBufferAppend(&dump, buf, strlen(buf));
6500:
6501: for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
6502: if( plrColumn(&plReader)!=iColumn ){
6503: iColumn = plrColumn(&plReader);
6504: sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
6505: assert( dump.nData>0 );
6506: dump.nData--; /* Overwrite trailing space. */
6507: assert( dump.pData[dump.nData]==' ');
6508: dataBufferAppend(&dump, buf, strlen(buf));
6509: }
6510: if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){
6511: sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ",
6512: plrPosition(&plReader),
6513: plrStartOffset(&plReader), plrEndOffset(&plReader));
6514: }else if( DL_DEFAULT==DL_POSITIONS ){
6515: sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader));
6516: }else{
6517: assert( NULL=="Unhandled DL_DEFAULT value");
6518: }
6519: dataBufferAppend(&dump, buf, strlen(buf));
6520: }
6521: plrDestroy(&plReader);
6522:
6523: assert( dump.nData>0 );
6524: dump.nData--; /* Overwrite trailing space. */
6525: assert( dump.pData[dump.nData]==' ');
6526: dataBufferAppend(&dump, "]] ", 3);
6527: }
6528: }
6529: dlrDestroy(&dlReader);
6530:
6531: assert( dump.nData>0 );
6532: dump.nData--; /* Overwrite trailing space. */
6533: assert( dump.pData[dump.nData]==' ');
6534: dump.pData[dump.nData] = '\0';
6535: assert( dump.nData>0 );
6536:
6537: /* Passes ownership of dump's buffer to pContext. */
6538: sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
6539: dump.pData = NULL;
6540: dump.nData = dump.nCapacity = 0;
6541: }
6542:
6543: /* Implements dump_doclist() for use in inspecting the fts2 index from
6544: ** tests. TEXT result containing a string representation of the
6545: ** doclist for the indicated term. dump_doclist(t, term, level, idx)
6546: ** dumps the doclist for term from the segment specified by level, idx
6547: ** (in %_segdir), while dump_doclist(t, term) dumps the logical
6548: ** doclist for the term across all segments. The per-segment doclist
6549: ** can contain deletions, while the full-index doclist will not
6550: ** (deletions are omitted).
6551: **
6552: ** Result formats differ with the setting of DL_DEFAULTS. Examples:
6553: **
6554: ** DL_DOCIDS: [1] [3] [7]
6555: ** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]]
6556: ** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]]
6557: **
6558: ** In each case the number after the outer '[' is the docid. In the
6559: ** latter two cases, the number before the inner '[' is the column
6560: ** associated with the values within. For DL_POSITIONS the numbers
6561: ** within are the positions, for DL_POSITIONS_OFFSETS they are the
6562: ** position, the start offset, and the end offset.
6563: */
6564: static void dumpDoclistFunc(
6565: sqlite3_context *pContext,
6566: int argc, sqlite3_value **argv
6567: ){
6568: fulltext_cursor *pCursor;
6569: if( argc!=2 && argc!=4 ){
6570: generateError(pContext, "dump_doclist", "incorrect arguments");
6571: }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
6572: sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
6573: generateError(pContext, "dump_doclist", "illegal first argument");
6574: }else if( sqlite3_value_text(argv[1])==NULL ||
6575: sqlite3_value_text(argv[1])[0]=='\0' ){
6576: generateError(pContext, "dump_doclist", "empty second argument");
6577: }else{
6578: const char *pTerm = (const char *)sqlite3_value_text(argv[1]);
6579: const int nTerm = strlen(pTerm);
6580: fulltext_vtab *v;
6581: int rc;
6582: DataBuffer doclist;
6583:
6584: memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
6585: v = cursor_vtab(pCursor);
6586:
6587: dataBufferInit(&doclist, 0);
6588:
6589: /* termSelect() yields the same logical doclist that queries are
6590: ** run against.
6591: */
6592: if( argc==2 ){
6593: rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist);
6594: }else{
6595: sqlite3_stmt *s = NULL;
6596:
6597: /* Get our specific segment's information. */
6598: rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
6599: if( rc==SQLITE_OK ){
6600: rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2]));
6601: if( rc==SQLITE_OK ){
6602: rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3]));
6603: }
6604: }
6605:
6606: if( rc==SQLITE_OK ){
6607: rc = sqlite3_step(s);
6608:
6609: if( rc==SQLITE_DONE ){
6610: dataBufferDestroy(&doclist);
6611: generateError(pContext, "dump_doclist", "segment not found");
6612: return;
6613: }
6614:
6615: /* Found a segment, load it into doclist. */
6616: if( rc==SQLITE_ROW ){
6617: const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
6618: const char *pData = sqlite3_column_blob(s, 2);
6619: const int nData = sqlite3_column_bytes(s, 2);
6620:
6621: /* loadSegment() is used by termSelect() to load each
6622: ** segment's data.
6623: */
6624: rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0,
6625: &doclist);
6626: if( rc==SQLITE_OK ){
6627: rc = sqlite3_step(s);
6628:
6629: /* Should not have more than one matching segment. */
6630: if( rc!=SQLITE_DONE ){
6631: sqlite3_reset(s);
6632: dataBufferDestroy(&doclist);
6633: generateError(pContext, "dump_doclist", "invalid segdir");
6634: return;
6635: }
6636: rc = SQLITE_OK;
6637: }
6638: }
6639: }
6640:
6641: sqlite3_reset(s);
6642: }
6643:
6644: if( rc==SQLITE_OK ){
6645: if( doclist.nData>0 ){
6646: createDoclistResult(pContext, doclist.pData, doclist.nData);
6647: }else{
6648: /* TODO(shess): This can happen if the term is not present, or
6649: ** if all instances of the term have been deleted and this is
6650: ** an all-index dump. It may be interesting to distinguish
6651: ** these cases.
6652: */
6653: sqlite3_result_text(pContext, "", 0, SQLITE_STATIC);
6654: }
6655: }else if( rc==SQLITE_NOMEM ){
6656: /* Handle out-of-memory cases specially because if they are
6657: ** generated in fts2 code they may not be reflected in the db
6658: ** handle.
6659: */
6660: /* TODO(shess): Handle this more comprehensively.
6661: ** sqlite3ErrStr() has what I need, but is internal.
6662: */
6663: generateError(pContext, "dump_doclist", "out of memory");
6664: }else{
6665: generateError(pContext, "dump_doclist", NULL);
6666: }
6667:
6668: dataBufferDestroy(&doclist);
6669: }
6670: }
6671: #endif
6672:
6673: /*
6674: ** This routine implements the xFindFunction method for the FTS2
6675: ** virtual table.
6676: */
6677: static int fulltextFindFunction(
6678: sqlite3_vtab *pVtab,
6679: int nArg,
6680: const char *zName,
6681: void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
6682: void **ppArg
6683: ){
6684: if( strcmp(zName,"snippet")==0 ){
6685: *pxFunc = snippetFunc;
6686: return 1;
6687: }else if( strcmp(zName,"offsets")==0 ){
6688: *pxFunc = snippetOffsetsFunc;
6689: return 1;
6690: }else if( strcmp(zName,"optimize")==0 ){
6691: *pxFunc = optimizeFunc;
6692: return 1;
6693: #ifdef SQLITE_TEST
6694: /* NOTE(shess): These functions are present only for testing
6695: ** purposes. No particular effort is made to optimize their
6696: ** execution or how they build their results.
6697: */
6698: }else if( strcmp(zName,"dump_terms")==0 ){
6699: /* fprintf(stderr, "Found dump_terms\n"); */
6700: *pxFunc = dumpTermsFunc;
6701: return 1;
6702: }else if( strcmp(zName,"dump_doclist")==0 ){
6703: /* fprintf(stderr, "Found dump_doclist\n"); */
6704: *pxFunc = dumpDoclistFunc;
6705: return 1;
6706: #endif
6707: }
6708: return 0;
6709: }
6710:
6711: /*
6712: ** Rename an fts2 table.
6713: */
6714: static int fulltextRename(
6715: sqlite3_vtab *pVtab,
6716: const char *zName
6717: ){
6718: fulltext_vtab *p = (fulltext_vtab *)pVtab;
6719: int rc = SQLITE_NOMEM;
6720: char *zSql = sqlite3_mprintf(
6721: "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
6722: "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
6723: "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
6724: , p->zDb, p->zName, zName
6725: , p->zDb, p->zName, zName
6726: , p->zDb, p->zName, zName
6727: );
6728: if( zSql ){
6729: rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
6730: sqlite3_free(zSql);
6731: }
6732: return rc;
6733: }
6734:
6735: static const sqlite3_module fts2Module = {
6736: /* iVersion */ 0,
6737: /* xCreate */ fulltextCreate,
6738: /* xConnect */ fulltextConnect,
6739: /* xBestIndex */ fulltextBestIndex,
6740: /* xDisconnect */ fulltextDisconnect,
6741: /* xDestroy */ fulltextDestroy,
6742: /* xOpen */ fulltextOpen,
6743: /* xClose */ fulltextClose,
6744: /* xFilter */ fulltextFilter,
6745: /* xNext */ fulltextNext,
6746: /* xEof */ fulltextEof,
6747: /* xColumn */ fulltextColumn,
6748: /* xRowid */ fulltextRowid,
6749: /* xUpdate */ fulltextUpdate,
6750: /* xBegin */ fulltextBegin,
6751: /* xSync */ fulltextSync,
6752: /* xCommit */ fulltextCommit,
6753: /* xRollback */ fulltextRollback,
6754: /* xFindFunction */ fulltextFindFunction,
6755: /* xRename */ fulltextRename,
6756: };
6757:
6758: static void hashDestroy(void *p){
6759: fts2Hash *pHash = (fts2Hash *)p;
6760: sqlite3Fts2HashClear(pHash);
6761: sqlite3_free(pHash);
6762: }
6763:
6764: /*
6765: ** The fts2 built-in tokenizers - "simple" and "porter" - are implemented
6766: ** in files fts2_tokenizer1.c and fts2_porter.c respectively. The following
6767: ** two forward declarations are for functions declared in these files
6768: ** used to retrieve the respective implementations.
6769: **
6770: ** Calling sqlite3Fts2SimpleTokenizerModule() sets the value pointed
6771: ** to by the argument to point a the "simple" tokenizer implementation.
6772: ** Function ...PorterTokenizerModule() sets *pModule to point to the
6773: ** porter tokenizer/stemmer implementation.
6774: */
6775: void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
6776: void sqlite3Fts2PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
6777: void sqlite3Fts2IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
6778:
6779: int sqlite3Fts2InitHashTable(sqlite3 *, fts2Hash *, const char *);
6780:
6781: /*
6782: ** Initialise the fts2 extension. If this extension is built as part
6783: ** of the sqlite library, then this function is called directly by
6784: ** SQLite. If fts2 is built as a dynamically loadable extension, this
6785: ** function is called by the sqlite3_extension_init() entry point.
6786: */
6787: int sqlite3Fts2Init(sqlite3 *db){
6788: int rc = SQLITE_OK;
6789: fts2Hash *pHash = 0;
6790: const sqlite3_tokenizer_module *pSimple = 0;
6791: const sqlite3_tokenizer_module *pPorter = 0;
6792: const sqlite3_tokenizer_module *pIcu = 0;
6793:
6794: sqlite3Fts2SimpleTokenizerModule(&pSimple);
6795: sqlite3Fts2PorterTokenizerModule(&pPorter);
6796: #ifdef SQLITE_ENABLE_ICU
6797: sqlite3Fts2IcuTokenizerModule(&pIcu);
6798: #endif
6799:
6800: /* Allocate and initialise the hash-table used to store tokenizers. */
6801: pHash = sqlite3_malloc(sizeof(fts2Hash));
6802: if( !pHash ){
6803: rc = SQLITE_NOMEM;
6804: }else{
6805: sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);
6806: }
6807:
6808: /* Load the built-in tokenizers into the hash table */
6809: if( rc==SQLITE_OK ){
6810: if( sqlite3Fts2HashInsert(pHash, "simple", 7, (void *)pSimple)
6811: || sqlite3Fts2HashInsert(pHash, "porter", 7, (void *)pPorter)
6812: || (pIcu && sqlite3Fts2HashInsert(pHash, "icu", 4, (void *)pIcu))
6813: ){
6814: rc = SQLITE_NOMEM;
6815: }
6816: }
6817:
6818: /* Create the virtual table wrapper around the hash-table and overload
6819: ** the two scalar functions. If this is successful, register the
6820: ** module with sqlite.
6821: */
6822: if( SQLITE_OK==rc
6823: && SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer"))
6824: && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
6825: && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
6826: && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
6827: #ifdef SQLITE_TEST
6828: && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1))
6829: && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1))
6830: #endif
6831: ){
6832: return sqlite3_create_module_v2(
6833: db, "fts2", &fts2Module, (void *)pHash, hashDestroy
6834: );
6835: }
6836:
6837: /* An error has occurred. Delete the hash table and return the error code. */
6838: assert( rc!=SQLITE_OK );
6839: if( pHash ){
6840: sqlite3Fts2HashClear(pHash);
6841: sqlite3_free(pHash);
6842: }
6843: return rc;
6844: }
6845:
6846: #if !SQLITE_CORE
6847: int sqlite3_extension_init(
6848: sqlite3 *db,
6849: char **pzErrMsg,
6850: const sqlite3_api_routines *pApi
6851: ){
6852: SQLITE_EXTENSION_INIT2(pApi)
6853: return sqlite3Fts2Init(db);
6854: }
6855: #endif
6856:
6857: #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts2.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts2