Annotation of embedaddon/sqlite3/ext/fts3/fts3_tokenizer1.c, revision 1.1.1.1
1.1 misho 1: /*
2: ** 2006 Oct 10
3: **
4: ** The author disclaims copyright to this source code. In place of
5: ** a legal notice, here is a blessing:
6: **
7: ** May you do good and not evil.
8: ** May you find forgiveness for yourself and forgive others.
9: ** May you share freely, never taking more than you give.
10: **
11: ******************************************************************************
12: **
13: ** Implementation of the "simple" full-text-search tokenizer.
14: */
15:
16: /*
17: ** The code in this file is only compiled if:
18: **
19: ** * The FTS3 module is being built as an extension
20: ** (in which case SQLITE_CORE is not defined), or
21: **
22: ** * The FTS3 module is being built into the core of
23: ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
24: */
25: #include "fts3Int.h"
26: #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
27:
28: #include <assert.h>
29: #include <stdlib.h>
30: #include <stdio.h>
31: #include <string.h>
32:
33: #include "fts3_tokenizer.h"
34:
35: typedef struct simple_tokenizer {
36: sqlite3_tokenizer base;
37: char delim[128]; /* flag ASCII delimiters */
38: } simple_tokenizer;
39:
40: typedef struct simple_tokenizer_cursor {
41: sqlite3_tokenizer_cursor base;
42: const char *pInput; /* input we are tokenizing */
43: int nBytes; /* size of the input */
44: int iOffset; /* current position in pInput */
45: int iToken; /* index of next token to be returned */
46: char *pToken; /* storage for current token */
47: int nTokenAllocated; /* space allocated to zToken buffer */
48: } simple_tokenizer_cursor;
49:
50:
51: static int simpleDelim(simple_tokenizer *t, unsigned char c){
52: return c<0x80 && t->delim[c];
53: }
54: static int fts3_isalnum(int x){
55: return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z');
56: }
57:
58: /*
59: ** Create a new tokenizer instance.
60: */
61: static int simpleCreate(
62: int argc, const char * const *argv,
63: sqlite3_tokenizer **ppTokenizer
64: ){
65: simple_tokenizer *t;
66:
67: t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
68: if( t==NULL ) return SQLITE_NOMEM;
69: memset(t, 0, sizeof(*t));
70:
71: /* TODO(shess) Delimiters need to remain the same from run to run,
72: ** else we need to reindex. One solution would be a meta-table to
73: ** track such information in the database, then we'd only want this
74: ** information on the initial create.
75: */
76: if( argc>1 ){
77: int i, n = (int)strlen(argv[1]);
78: for(i=0; i<n; i++){
79: unsigned char ch = argv[1][i];
80: /* We explicitly don't support UTF-8 delimiters for now. */
81: if( ch>=0x80 ){
82: sqlite3_free(t);
83: return SQLITE_ERROR;
84: }
85: t->delim[ch] = 1;
86: }
87: } else {
88: /* Mark non-alphanumeric ASCII characters as delimiters */
89: int i;
90: for(i=1; i<0x80; i++){
91: t->delim[i] = !fts3_isalnum(i) ? -1 : 0;
92: }
93: }
94:
95: *ppTokenizer = &t->base;
96: return SQLITE_OK;
97: }
98:
99: /*
100: ** Destroy a tokenizer
101: */
102: static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
103: sqlite3_free(pTokenizer);
104: return SQLITE_OK;
105: }
106:
107: /*
108: ** Prepare to begin tokenizing a particular string. The input
109: ** string to be tokenized is pInput[0..nBytes-1]. A cursor
110: ** used to incrementally tokenize this string is returned in
111: ** *ppCursor.
112: */
113: static int simpleOpen(
114: sqlite3_tokenizer *pTokenizer, /* The tokenizer */
115: const char *pInput, int nBytes, /* String to be tokenized */
116: sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
117: ){
118: simple_tokenizer_cursor *c;
119:
120: UNUSED_PARAMETER(pTokenizer);
121:
122: c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
123: if( c==NULL ) return SQLITE_NOMEM;
124:
125: c->pInput = pInput;
126: if( pInput==0 ){
127: c->nBytes = 0;
128: }else if( nBytes<0 ){
129: c->nBytes = (int)strlen(pInput);
130: }else{
131: c->nBytes = nBytes;
132: }
133: c->iOffset = 0; /* start tokenizing at the beginning */
134: c->iToken = 0;
135: c->pToken = NULL; /* no space allocated, yet. */
136: c->nTokenAllocated = 0;
137:
138: *ppCursor = &c->base;
139: return SQLITE_OK;
140: }
141:
142: /*
143: ** Close a tokenization cursor previously opened by a call to
144: ** simpleOpen() above.
145: */
146: static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
147: simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
148: sqlite3_free(c->pToken);
149: sqlite3_free(c);
150: return SQLITE_OK;
151: }
152:
153: /*
154: ** Extract the next token from a tokenization cursor. The cursor must
155: ** have been opened by a prior call to simpleOpen().
156: */
157: static int simpleNext(
158: sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
159: const char **ppToken, /* OUT: *ppToken is the token text */
160: int *pnBytes, /* OUT: Number of bytes in token */
161: int *piStartOffset, /* OUT: Starting offset of token */
162: int *piEndOffset, /* OUT: Ending offset of token */
163: int *piPosition /* OUT: Position integer of token */
164: ){
165: simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
166: simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
167: unsigned char *p = (unsigned char *)c->pInput;
168:
169: while( c->iOffset<c->nBytes ){
170: int iStartOffset;
171:
172: /* Scan past delimiter characters */
173: while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
174: c->iOffset++;
175: }
176:
177: /* Count non-delimiter characters. */
178: iStartOffset = c->iOffset;
179: while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
180: c->iOffset++;
181: }
182:
183: if( c->iOffset>iStartOffset ){
184: int i, n = c->iOffset-iStartOffset;
185: if( n>c->nTokenAllocated ){
186: char *pNew;
187: c->nTokenAllocated = n+20;
188: pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
189: if( !pNew ) return SQLITE_NOMEM;
190: c->pToken = pNew;
191: }
192: for(i=0; i<n; i++){
193: /* TODO(shess) This needs expansion to handle UTF-8
194: ** case-insensitivity.
195: */
196: unsigned char ch = p[iStartOffset+i];
197: c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch);
198: }
199: *ppToken = c->pToken;
200: *pnBytes = n;
201: *piStartOffset = iStartOffset;
202: *piEndOffset = c->iOffset;
203: *piPosition = c->iToken++;
204:
205: return SQLITE_OK;
206: }
207: }
208: return SQLITE_DONE;
209: }
210:
211: /*
212: ** The set of routines that implement the simple tokenizer
213: */
214: static const sqlite3_tokenizer_module simpleTokenizerModule = {
215: 0,
216: simpleCreate,
217: simpleDestroy,
218: simpleOpen,
219: simpleClose,
220: simpleNext,
221: };
222:
223: /*
224: ** Allocate a new simple tokenizer. Return a pointer to the new
225: ** tokenizer in *ppModule
226: */
227: void sqlite3Fts3SimpleTokenizerModule(
228: sqlite3_tokenizer_module const**ppModule
229: ){
230: *ppModule = &simpleTokenizerModule;
231: }
232:
233: #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts3_tokenizer1.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts3