Annotation of embedaddon/sqlite3/ext/fts2/fts2_tokenizer1.c, revision 1.1.1.1
1.1 misho 1: /*
2: ** 2006 Oct 10
3: **
4: ** The author disclaims copyright to this source code. In place of
5: ** a legal notice, here is a blessing:
6: **
7: ** May you do good and not evil.
8: ** May you find forgiveness for yourself and forgive others.
9: ** May you share freely, never taking more than you give.
10: **
11: ******************************************************************************
12: **
13: ** Implementation of the "simple" full-text-search tokenizer.
14: */
15:
16: /*
17: ** The code in this file is only compiled if:
18: **
19: ** * The FTS2 module is being built as an extension
20: ** (in which case SQLITE_CORE is not defined), or
21: **
22: ** * The FTS2 module is being built into the core of
23: ** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
24: */
25: #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
26:
27:
28: #include <assert.h>
29: #include <stdlib.h>
30: #include <stdio.h>
31: #include <string.h>
32:
33: #include "fts2_tokenizer.h"
34:
35: typedef struct simple_tokenizer {
36: sqlite3_tokenizer base;
37: char delim[128]; /* flag ASCII delimiters */
38: } simple_tokenizer;
39:
40: typedef struct simple_tokenizer_cursor {
41: sqlite3_tokenizer_cursor base;
42: const char *pInput; /* input we are tokenizing */
43: int nBytes; /* size of the input */
44: int iOffset; /* current position in pInput */
45: int iToken; /* index of next token to be returned */
46: char *pToken; /* storage for current token */
47: int nTokenAllocated; /* space allocated to zToken buffer */
48: } simple_tokenizer_cursor;
49:
50:
51: /* Forward declaration */
52: static const sqlite3_tokenizer_module simpleTokenizerModule;
53:
54: static int simpleDelim(simple_tokenizer *t, unsigned char c){
55: return c<0x80 && t->delim[c];
56: }
57:
58: /*
59: ** Create a new tokenizer instance.
60: */
61: static int simpleCreate(
62: int argc, const char * const *argv,
63: sqlite3_tokenizer **ppTokenizer
64: ){
65: simple_tokenizer *t;
66:
67: t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
68: if( t==NULL ) return SQLITE_NOMEM;
69: memset(t, 0, sizeof(*t));
70:
71: /* TODO(shess) Delimiters need to remain the same from run to run,
72: ** else we need to reindex. One solution would be a meta-table to
73: ** track such information in the database, then we'd only want this
74: ** information on the initial create.
75: */
76: if( argc>1 ){
77: int i, n = strlen(argv[1]);
78: for(i=0; i<n; i++){
79: unsigned char ch = argv[1][i];
80: /* We explicitly don't support UTF-8 delimiters for now. */
81: if( ch>=0x80 ){
82: sqlite3_free(t);
83: return SQLITE_ERROR;
84: }
85: t->delim[ch] = 1;
86: }
87: } else {
88: /* Mark non-alphanumeric ASCII characters as delimiters */
89: int i;
90: for(i=1; i<0x80; i++){
91: t->delim[i] = !((i>='0' && i<='9') || (i>='A' && i<='Z') ||
92: (i>='a' && i<='z'));
93: }
94: }
95:
96: *ppTokenizer = &t->base;
97: return SQLITE_OK;
98: }
99:
100: /*
101: ** Destroy a tokenizer
102: */
103: static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
104: sqlite3_free(pTokenizer);
105: return SQLITE_OK;
106: }
107:
108: /*
109: ** Prepare to begin tokenizing a particular string. The input
110: ** string to be tokenized is pInput[0..nBytes-1]. A cursor
111: ** used to incrementally tokenize this string is returned in
112: ** *ppCursor.
113: */
114: static int simpleOpen(
115: sqlite3_tokenizer *pTokenizer, /* The tokenizer */
116: const char *pInput, int nBytes, /* String to be tokenized */
117: sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
118: ){
119: simple_tokenizer_cursor *c;
120:
121: c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
122: if( c==NULL ) return SQLITE_NOMEM;
123:
124: c->pInput = pInput;
125: if( pInput==0 ){
126: c->nBytes = 0;
127: }else if( nBytes<0 ){
128: c->nBytes = (int)strlen(pInput);
129: }else{
130: c->nBytes = nBytes;
131: }
132: c->iOffset = 0; /* start tokenizing at the beginning */
133: c->iToken = 0;
134: c->pToken = NULL; /* no space allocated, yet. */
135: c->nTokenAllocated = 0;
136:
137: *ppCursor = &c->base;
138: return SQLITE_OK;
139: }
140:
141: /*
142: ** Close a tokenization cursor previously opened by a call to
143: ** simpleOpen() above.
144: */
145: static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
146: simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
147: sqlite3_free(c->pToken);
148: sqlite3_free(c);
149: return SQLITE_OK;
150: }
151:
152: /*
153: ** Extract the next token from a tokenization cursor. The cursor must
154: ** have been opened by a prior call to simpleOpen().
155: */
156: static int simpleNext(
157: sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
158: const char **ppToken, /* OUT: *ppToken is the token text */
159: int *pnBytes, /* OUT: Number of bytes in token */
160: int *piStartOffset, /* OUT: Starting offset of token */
161: int *piEndOffset, /* OUT: Ending offset of token */
162: int *piPosition /* OUT: Position integer of token */
163: ){
164: simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
165: simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
166: unsigned char *p = (unsigned char *)c->pInput;
167:
168: while( c->iOffset<c->nBytes ){
169: int iStartOffset;
170:
171: /* Scan past delimiter characters */
172: while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
173: c->iOffset++;
174: }
175:
176: /* Count non-delimiter characters. */
177: iStartOffset = c->iOffset;
178: while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
179: c->iOffset++;
180: }
181:
182: if( c->iOffset>iStartOffset ){
183: int i, n = c->iOffset-iStartOffset;
184: if( n>c->nTokenAllocated ){
185: c->nTokenAllocated = n+20;
186: c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
187: if( c->pToken==NULL ) return SQLITE_NOMEM;
188: }
189: for(i=0; i<n; i++){
190: /* TODO(shess) This needs expansion to handle UTF-8
191: ** case-insensitivity.
192: */
193: unsigned char ch = p[iStartOffset+i];
194: c->pToken[i] = (ch>='A' && ch<='Z') ? (ch - 'A' + 'a') : ch;
195: }
196: *ppToken = c->pToken;
197: *pnBytes = n;
198: *piStartOffset = iStartOffset;
199: *piEndOffset = c->iOffset;
200: *piPosition = c->iToken++;
201:
202: return SQLITE_OK;
203: }
204: }
205: return SQLITE_DONE;
206: }
207:
208: /*
209: ** The set of routines that implement the simple tokenizer
210: */
211: static const sqlite3_tokenizer_module simpleTokenizerModule = {
212: 0,
213: simpleCreate,
214: simpleDestroy,
215: simpleOpen,
216: simpleClose,
217: simpleNext,
218: };
219:
220: /*
221: ** Allocate a new simple tokenizer. Return a pointer to the new
222: ** tokenizer in *ppModule
223: */
224: void sqlite3Fts2SimpleTokenizerModule(
225: sqlite3_tokenizer_module const**ppModule
226: ){
227: *ppModule = &simpleTokenizerModule;
228: }
229:
230: #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts2_tokenizer1.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts2