Annotation of embedaddon/sqlite3/ext/fts1/fts1_tokenizer1.c, revision 1.1.1.1
1.1 misho 1: /*
2: ** The author disclaims copyright to this source code.
3: **
4: *************************************************************************
5: ** Implementation of the "simple" full-text-search tokenizer.
6: */
7:
8: /*
9: ** The code in this file is only compiled if:
10: **
11: ** * The FTS1 module is being built as an extension
12: ** (in which case SQLITE_CORE is not defined), or
13: **
14: ** * The FTS1 module is being built into the core of
15: ** SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
16: */
17: #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
18:
19:
20: #include <assert.h>
21: #include <stdlib.h>
22: #include <stdio.h>
23: #include <string.h>
24: #include <ctype.h>
25:
26: #include "fts1_tokenizer.h"
27:
28: typedef struct simple_tokenizer {
29: sqlite3_tokenizer base;
30: char delim[128]; /* flag ASCII delimiters */
31: } simple_tokenizer;
32:
33: typedef struct simple_tokenizer_cursor {
34: sqlite3_tokenizer_cursor base;
35: const char *pInput; /* input we are tokenizing */
36: int nBytes; /* size of the input */
37: int iOffset; /* current position in pInput */
38: int iToken; /* index of next token to be returned */
39: char *pToken; /* storage for current token */
40: int nTokenAllocated; /* space allocated to zToken buffer */
41: } simple_tokenizer_cursor;
42:
43:
44: /* Forward declaration */
45: static const sqlite3_tokenizer_module simpleTokenizerModule;
46:
47: static int isDelim(simple_tokenizer *t, unsigned char c){
48: return c<0x80 && t->delim[c];
49: }
50:
51: /*
52: ** Create a new tokenizer instance.
53: */
54: static int simpleCreate(
55: int argc, const char * const *argv,
56: sqlite3_tokenizer **ppTokenizer
57: ){
58: simple_tokenizer *t;
59:
60: t = (simple_tokenizer *) calloc(sizeof(*t), 1);
61: if( t==NULL ) return SQLITE_NOMEM;
62:
63: /* TODO(shess) Delimiters need to remain the same from run to run,
64: ** else we need to reindex. One solution would be a meta-table to
65: ** track such information in the database, then we'd only want this
66: ** information on the initial create.
67: */
68: if( argc>1 ){
69: int i, n = strlen(argv[1]);
70: for(i=0; i<n; i++){
71: unsigned char ch = argv[1][i];
72: /* We explicitly don't support UTF-8 delimiters for now. */
73: if( ch>=0x80 ){
74: free(t);
75: return SQLITE_ERROR;
76: }
77: t->delim[ch] = 1;
78: }
79: } else {
80: /* Mark non-alphanumeric ASCII characters as delimiters */
81: int i;
82: for(i=1; i<0x80; i++){
83: t->delim[i] = !isalnum(i);
84: }
85: }
86:
87: *ppTokenizer = &t->base;
88: return SQLITE_OK;
89: }
90:
91: /*
92: ** Destroy a tokenizer
93: */
94: static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
95: free(pTokenizer);
96: return SQLITE_OK;
97: }
98:
99: /*
100: ** Prepare to begin tokenizing a particular string. The input
101: ** string to be tokenized is pInput[0..nBytes-1]. A cursor
102: ** used to incrementally tokenize this string is returned in
103: ** *ppCursor.
104: */
105: static int simpleOpen(
106: sqlite3_tokenizer *pTokenizer, /* The tokenizer */
107: const char *pInput, int nBytes, /* String to be tokenized */
108: sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
109: ){
110: simple_tokenizer_cursor *c;
111:
112: c = (simple_tokenizer_cursor *) malloc(sizeof(*c));
113: if( c==NULL ) return SQLITE_NOMEM;
114:
115: c->pInput = pInput;
116: if( pInput==0 ){
117: c->nBytes = 0;
118: }else if( nBytes<0 ){
119: c->nBytes = (int)strlen(pInput);
120: }else{
121: c->nBytes = nBytes;
122: }
123: c->iOffset = 0; /* start tokenizing at the beginning */
124: c->iToken = 0;
125: c->pToken = NULL; /* no space allocated, yet. */
126: c->nTokenAllocated = 0;
127:
128: *ppCursor = &c->base;
129: return SQLITE_OK;
130: }
131:
132: /*
133: ** Close a tokenization cursor previously opened by a call to
134: ** simpleOpen() above.
135: */
136: static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
137: simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
138: free(c->pToken);
139: free(c);
140: return SQLITE_OK;
141: }
142:
143: /*
144: ** Extract the next token from a tokenization cursor. The cursor must
145: ** have been opened by a prior call to simpleOpen().
146: */
147: static int simpleNext(
148: sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
149: const char **ppToken, /* OUT: *ppToken is the token text */
150: int *pnBytes, /* OUT: Number of bytes in token */
151: int *piStartOffset, /* OUT: Starting offset of token */
152: int *piEndOffset, /* OUT: Ending offset of token */
153: int *piPosition /* OUT: Position integer of token */
154: ){
155: simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
156: simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
157: unsigned char *p = (unsigned char *)c->pInput;
158:
159: while( c->iOffset<c->nBytes ){
160: int iStartOffset;
161:
162: /* Scan past delimiter characters */
163: while( c->iOffset<c->nBytes && isDelim(t, p[c->iOffset]) ){
164: c->iOffset++;
165: }
166:
167: /* Count non-delimiter characters. */
168: iStartOffset = c->iOffset;
169: while( c->iOffset<c->nBytes && !isDelim(t, p[c->iOffset]) ){
170: c->iOffset++;
171: }
172:
173: if( c->iOffset>iStartOffset ){
174: int i, n = c->iOffset-iStartOffset;
175: if( n>c->nTokenAllocated ){
176: c->nTokenAllocated = n+20;
177: c->pToken = realloc(c->pToken, c->nTokenAllocated);
178: if( c->pToken==NULL ) return SQLITE_NOMEM;
179: }
180: for(i=0; i<n; i++){
181: /* TODO(shess) This needs expansion to handle UTF-8
182: ** case-insensitivity.
183: */
184: unsigned char ch = p[iStartOffset+i];
185: c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
186: }
187: *ppToken = c->pToken;
188: *pnBytes = n;
189: *piStartOffset = iStartOffset;
190: *piEndOffset = c->iOffset;
191: *piPosition = c->iToken++;
192:
193: return SQLITE_OK;
194: }
195: }
196: return SQLITE_DONE;
197: }
198:
199: /*
200: ** The set of routines that implement the simple tokenizer
201: */
202: static const sqlite3_tokenizer_module simpleTokenizerModule = {
203: 0,
204: simpleCreate,
205: simpleDestroy,
206: simpleOpen,
207: simpleClose,
208: simpleNext,
209: };
210:
211: /*
212: ** Allocate a new simple tokenizer. Return a pointer to the new
213: ** tokenizer in *ppModule
214: */
215: void sqlite3Fts1SimpleTokenizerModule(
216: sqlite3_tokenizer_module const**ppModule
217: ){
218: *ppModule = &simpleTokenizerModule;
219: }
220:
221: #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts1_tokenizer1.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts1