Annotation of embedaddon/sqlite3/ext/fts3/fts3_tokenizer.c, revision 1.1.1.1
1.1 misho 1: /*
2: ** 2007 June 22
3: **
4: ** The author disclaims copyright to this source code. In place of
5: ** a legal notice, here is a blessing:
6: **
7: ** May you do good and not evil.
8: ** May you find forgiveness for yourself and forgive others.
9: ** May you share freely, never taking more than you give.
10: **
11: ******************************************************************************
12: **
13: ** This is part of an SQLite module implementing full-text search.
14: ** This particular file implements the generic tokenizer interface.
15: */
16:
17: /*
18: ** The code in this file is only compiled if:
19: **
20: ** * The FTS3 module is being built as an extension
21: ** (in which case SQLITE_CORE is not defined), or
22: **
23: ** * The FTS3 module is being built into the core of
24: ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
25: */
26: #include "fts3Int.h"
27: #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
28:
29: #include <assert.h>
30: #include <string.h>
31:
32: /*
33: ** Implementation of the SQL scalar function for accessing the underlying
34: ** hash table. This function may be called as follows:
35: **
36: ** SELECT <function-name>(<key-name>);
37: ** SELECT <function-name>(<key-name>, <pointer>);
38: **
39: ** where <function-name> is the name passed as the second argument
40: ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
41: **
42: ** If the <pointer> argument is specified, it must be a blob value
43: ** containing a pointer to be stored as the hash data corresponding
44: ** to the string <key-name>. If <pointer> is not specified, then
45: ** the string <key-name> must already exist in the has table. Otherwise,
46: ** an error is returned.
47: **
48: ** Whether or not the <pointer> argument is specified, the value returned
49: ** is a blob containing the pointer stored as the hash data corresponding
50: ** to string <key-name> (after the hash-table is updated, if applicable).
51: */
52: static void scalarFunc(
53: sqlite3_context *context,
54: int argc,
55: sqlite3_value **argv
56: ){
57: Fts3Hash *pHash;
58: void *pPtr = 0;
59: const unsigned char *zName;
60: int nName;
61:
62: assert( argc==1 || argc==2 );
63:
64: pHash = (Fts3Hash *)sqlite3_user_data(context);
65:
66: zName = sqlite3_value_text(argv[0]);
67: nName = sqlite3_value_bytes(argv[0])+1;
68:
69: if( argc==2 ){
70: void *pOld;
71: int n = sqlite3_value_bytes(argv[1]);
72: if( n!=sizeof(pPtr) ){
73: sqlite3_result_error(context, "argument type mismatch", -1);
74: return;
75: }
76: pPtr = *(void **)sqlite3_value_blob(argv[1]);
77: pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
78: if( pOld==pPtr ){
79: sqlite3_result_error(context, "out of memory", -1);
80: return;
81: }
82: }else{
83: pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
84: if( !pPtr ){
85: char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
86: sqlite3_result_error(context, zErr, -1);
87: sqlite3_free(zErr);
88: return;
89: }
90: }
91:
92: sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
93: }
94:
95: int sqlite3Fts3IsIdChar(char c){
96: static const char isFtsIdChar[] = {
97: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
98: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
99: 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
100: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
101: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
102: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
103: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
104: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
105: };
106: return (c&0x80 || isFtsIdChar[(int)(c)]);
107: }
108:
109: const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
110: const char *z1;
111: const char *z2 = 0;
112:
113: /* Find the start of the next token. */
114: z1 = zStr;
115: while( z2==0 ){
116: char c = *z1;
117: switch( c ){
118: case '\0': return 0; /* No more tokens here */
119: case '\'':
120: case '"':
121: case '`': {
122: z2 = z1;
123: while( *++z2 && (*z2!=c || *++z2==c) );
124: break;
125: }
126: case '[':
127: z2 = &z1[1];
128: while( *z2 && z2[0]!=']' ) z2++;
129: if( *z2 ) z2++;
130: break;
131:
132: default:
133: if( sqlite3Fts3IsIdChar(*z1) ){
134: z2 = &z1[1];
135: while( sqlite3Fts3IsIdChar(*z2) ) z2++;
136: }else{
137: z1++;
138: }
139: }
140: }
141:
142: *pn = (int)(z2-z1);
143: return z1;
144: }
145:
146: int sqlite3Fts3InitTokenizer(
147: Fts3Hash *pHash, /* Tokenizer hash table */
148: const char *zArg, /* Tokenizer name */
149: sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */
150: char **pzErr /* OUT: Set to malloced error message */
151: ){
152: int rc;
153: char *z = (char *)zArg;
154: int n = 0;
155: char *zCopy;
156: char *zEnd; /* Pointer to nul-term of zCopy */
157: sqlite3_tokenizer_module *m;
158:
159: zCopy = sqlite3_mprintf("%s", zArg);
160: if( !zCopy ) return SQLITE_NOMEM;
161: zEnd = &zCopy[strlen(zCopy)];
162:
163: z = (char *)sqlite3Fts3NextToken(zCopy, &n);
164: z[n] = '\0';
165: sqlite3Fts3Dequote(z);
166:
167: m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
168: if( !m ){
169: *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
170: rc = SQLITE_ERROR;
171: }else{
172: char const **aArg = 0;
173: int iArg = 0;
174: z = &z[n+1];
175: while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
176: int nNew = sizeof(char *)*(iArg+1);
177: char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
178: if( !aNew ){
179: sqlite3_free(zCopy);
180: sqlite3_free((void *)aArg);
181: return SQLITE_NOMEM;
182: }
183: aArg = aNew;
184: aArg[iArg++] = z;
185: z[n] = '\0';
186: sqlite3Fts3Dequote(z);
187: z = &z[n+1];
188: }
189: rc = m->xCreate(iArg, aArg, ppTok);
190: assert( rc!=SQLITE_OK || *ppTok );
191: if( rc!=SQLITE_OK ){
192: *pzErr = sqlite3_mprintf("unknown tokenizer");
193: }else{
194: (*ppTok)->pModule = m;
195: }
196: sqlite3_free((void *)aArg);
197: }
198:
199: sqlite3_free(zCopy);
200: return rc;
201: }
202:
203:
204: #ifdef SQLITE_TEST
205:
206: #include <tcl.h>
207: #include <string.h>
208:
209: /*
210: ** Implementation of a special SQL scalar function for testing tokenizers
211: ** designed to be used in concert with the Tcl testing framework. This
212: ** function must be called with two arguments:
213: **
214: ** SELECT <function-name>(<key-name>, <input-string>);
215: ** SELECT <function-name>(<key-name>, <pointer>);
216: **
217: ** where <function-name> is the name passed as the second argument
218: ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
219: ** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
220: **
221: ** The return value is a string that may be interpreted as a Tcl
222: ** list. For each token in the <input-string>, three elements are
223: ** added to the returned list. The first is the token position, the
224: ** second is the token text (folded, stemmed, etc.) and the third is the
225: ** substring of <input-string> associated with the token. For example,
226: ** using the built-in "simple" tokenizer:
227: **
228: ** SELECT fts_tokenizer_test('simple', 'I don't see how');
229: **
230: ** will return the string:
231: **
232: ** "{0 i I 1 dont don't 2 see see 3 how how}"
233: **
234: */
235: static void testFunc(
236: sqlite3_context *context,
237: int argc,
238: sqlite3_value **argv
239: ){
240: Fts3Hash *pHash;
241: sqlite3_tokenizer_module *p;
242: sqlite3_tokenizer *pTokenizer = 0;
243: sqlite3_tokenizer_cursor *pCsr = 0;
244:
245: const char *zErr = 0;
246:
247: const char *zName;
248: int nName;
249: const char *zInput;
250: int nInput;
251:
252: const char *zArg = 0;
253:
254: const char *zToken;
255: int nToken;
256: int iStart;
257: int iEnd;
258: int iPos;
259:
260: Tcl_Obj *pRet;
261:
262: assert( argc==2 || argc==3 );
263:
264: nName = sqlite3_value_bytes(argv[0]);
265: zName = (const char *)sqlite3_value_text(argv[0]);
266: nInput = sqlite3_value_bytes(argv[argc-1]);
267: zInput = (const char *)sqlite3_value_text(argv[argc-1]);
268:
269: if( argc==3 ){
270: zArg = (const char *)sqlite3_value_text(argv[1]);
271: }
272:
273: pHash = (Fts3Hash *)sqlite3_user_data(context);
274: p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
275:
276: if( !p ){
277: char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
278: sqlite3_result_error(context, zErr, -1);
279: sqlite3_free(zErr);
280: return;
281: }
282:
283: pRet = Tcl_NewObj();
284: Tcl_IncrRefCount(pRet);
285:
286: if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
287: zErr = "error in xCreate()";
288: goto finish;
289: }
290: pTokenizer->pModule = p;
291: if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
292: zErr = "error in xOpen()";
293: goto finish;
294: }
295: pCsr->pTokenizer = pTokenizer;
296:
297: while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
298: Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
299: Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
300: zToken = &zInput[iStart];
301: nToken = iEnd-iStart;
302: Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
303: }
304:
305: if( SQLITE_OK!=p->xClose(pCsr) ){
306: zErr = "error in xClose()";
307: goto finish;
308: }
309: if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
310: zErr = "error in xDestroy()";
311: goto finish;
312: }
313:
314: finish:
315: if( zErr ){
316: sqlite3_result_error(context, zErr, -1);
317: }else{
318: sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
319: }
320: Tcl_DecrRefCount(pRet);
321: }
322:
323: static
324: int registerTokenizer(
325: sqlite3 *db,
326: char *zName,
327: const sqlite3_tokenizer_module *p
328: ){
329: int rc;
330: sqlite3_stmt *pStmt;
331: const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
332:
333: rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
334: if( rc!=SQLITE_OK ){
335: return rc;
336: }
337:
338: sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
339: sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
340: sqlite3_step(pStmt);
341:
342: return sqlite3_finalize(pStmt);
343: }
344:
345: static
346: int queryTokenizer(
347: sqlite3 *db,
348: char *zName,
349: const sqlite3_tokenizer_module **pp
350: ){
351: int rc;
352: sqlite3_stmt *pStmt;
353: const char zSql[] = "SELECT fts3_tokenizer(?)";
354:
355: *pp = 0;
356: rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
357: if( rc!=SQLITE_OK ){
358: return rc;
359: }
360:
361: sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
362: if( SQLITE_ROW==sqlite3_step(pStmt) ){
363: if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
364: memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
365: }
366: }
367:
368: return sqlite3_finalize(pStmt);
369: }
370:
371: void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
372:
373: /*
374: ** Implementation of the scalar function fts3_tokenizer_internal_test().
375: ** This function is used for testing only, it is not included in the
376: ** build unless SQLITE_TEST is defined.
377: **
378: ** The purpose of this is to test that the fts3_tokenizer() function
379: ** can be used as designed by the C-code in the queryTokenizer and
380: ** registerTokenizer() functions above. These two functions are repeated
381: ** in the README.tokenizer file as an example, so it is important to
382: ** test them.
383: **
384: ** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
385: ** function with no arguments. An assert() will fail if a problem is
386: ** detected. i.e.:
387: **
388: ** SELECT fts3_tokenizer_internal_test();
389: **
390: */
391: static void intTestFunc(
392: sqlite3_context *context,
393: int argc,
394: sqlite3_value **argv
395: ){
396: int rc;
397: const sqlite3_tokenizer_module *p1;
398: const sqlite3_tokenizer_module *p2;
399: sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
400:
401: UNUSED_PARAMETER(argc);
402: UNUSED_PARAMETER(argv);
403:
404: /* Test the query function */
405: sqlite3Fts3SimpleTokenizerModule(&p1);
406: rc = queryTokenizer(db, "simple", &p2);
407: assert( rc==SQLITE_OK );
408: assert( p1==p2 );
409: rc = queryTokenizer(db, "nosuchtokenizer", &p2);
410: assert( rc==SQLITE_ERROR );
411: assert( p2==0 );
412: assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
413:
414: /* Test the storage function */
415: rc = registerTokenizer(db, "nosuchtokenizer", p1);
416: assert( rc==SQLITE_OK );
417: rc = queryTokenizer(db, "nosuchtokenizer", &p2);
418: assert( rc==SQLITE_OK );
419: assert( p2==p1 );
420:
421: sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
422: }
423:
424: #endif
425:
426: /*
427: ** Set up SQL objects in database db used to access the contents of
428: ** the hash table pointed to by argument pHash. The hash table must
429: ** been initialised to use string keys, and to take a private copy
430: ** of the key when a value is inserted. i.e. by a call similar to:
431: **
432: ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
433: **
434: ** This function adds a scalar function (see header comment above
435: ** scalarFunc() in this file for details) and, if ENABLE_TABLE is
436: ** defined at compilation time, a temporary virtual table (see header
437: ** comment above struct HashTableVtab) to the database schema. Both
438: ** provide read/write access to the contents of *pHash.
439: **
440: ** The third argument to this function, zName, is used as the name
441: ** of both the scalar and, if created, the virtual table.
442: */
443: int sqlite3Fts3InitHashTable(
444: sqlite3 *db,
445: Fts3Hash *pHash,
446: const char *zName
447: ){
448: int rc = SQLITE_OK;
449: void *p = (void *)pHash;
450: const int any = SQLITE_ANY;
451:
452: #ifdef SQLITE_TEST
453: char *zTest = 0;
454: char *zTest2 = 0;
455: void *pdb = (void *)db;
456: zTest = sqlite3_mprintf("%s_test", zName);
457: zTest2 = sqlite3_mprintf("%s_internal_test", zName);
458: if( !zTest || !zTest2 ){
459: rc = SQLITE_NOMEM;
460: }
461: #endif
462:
463: if( SQLITE_OK==rc ){
464: rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
465: }
466: if( SQLITE_OK==rc ){
467: rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
468: }
469: #ifdef SQLITE_TEST
470: if( SQLITE_OK==rc ){
471: rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0);
472: }
473: if( SQLITE_OK==rc ){
474: rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0);
475: }
476: if( SQLITE_OK==rc ){
477: rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);
478: }
479: #endif
480:
481: #ifdef SQLITE_TEST
482: sqlite3_free(zTest);
483: sqlite3_free(zTest2);
484: #endif
485:
486: return rc;
487: }
488:
489: #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts3_tokenizer.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts3