Annotation of embedaddon/sqlite3/ext/fts2/fts2_tokenizer.c, revision 1.1.1.1
1.1 misho 1: /*
2: ** 2007 June 22
3: **
4: ** The author disclaims copyright to this source code. In place of
5: ** a legal notice, here is a blessing:
6: **
7: ** May you do good and not evil.
8: ** May you find forgiveness for yourself and forgive others.
9: ** May you share freely, never taking more than you give.
10: **
11: ******************************************************************************
12: **
13: ** This is part of an SQLite module implementing full-text search.
14: ** This particular file implements the generic tokenizer interface.
15: */
16:
17: /*
18: ** The code in this file is only compiled if:
19: **
20: ** * The FTS2 module is being built as an extension
21: ** (in which case SQLITE_CORE is not defined), or
22: **
23: ** * The FTS2 module is being built into the core of
24: ** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
25: */
26: #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
27:
28:
29: #include "sqlite3.h"
30: #include "sqlite3ext.h"
31: SQLITE_EXTENSION_INIT1
32:
33: #include "fts2_hash.h"
34: #include "fts2_tokenizer.h"
35: #include <assert.h>
36:
37: /*
38: ** Implementation of the SQL scalar function for accessing the underlying
39: ** hash table. This function may be called as follows:
40: **
41: ** SELECT <function-name>(<key-name>);
42: ** SELECT <function-name>(<key-name>, <pointer>);
43: **
44: ** where <function-name> is the name passed as the second argument
45: ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer').
46: **
47: ** If the <pointer> argument is specified, it must be a blob value
48: ** containing a pointer to be stored as the hash data corresponding
49: ** to the string <key-name>. If <pointer> is not specified, then
50: ** the string <key-name> must already exist in the has table. Otherwise,
51: ** an error is returned.
52: **
53: ** Whether or not the <pointer> argument is specified, the value returned
54: ** is a blob containing the pointer stored as the hash data corresponding
55: ** to string <key-name> (after the hash-table is updated, if applicable).
56: */
57: static void scalarFunc(
58: sqlite3_context *context,
59: int argc,
60: sqlite3_value **argv
61: ){
62: fts2Hash *pHash;
63: void *pPtr = 0;
64: const unsigned char *zName;
65: int nName;
66:
67: assert( argc==1 || argc==2 );
68:
69: pHash = (fts2Hash *)sqlite3_user_data(context);
70:
71: zName = sqlite3_value_text(argv[0]);
72: nName = sqlite3_value_bytes(argv[0])+1;
73:
74: if( argc==2 ){
75: void *pOld;
76: int n = sqlite3_value_bytes(argv[1]);
77: if( n!=sizeof(pPtr) ){
78: sqlite3_result_error(context, "argument type mismatch", -1);
79: return;
80: }
81: pPtr = *(void **)sqlite3_value_blob(argv[1]);
82: pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr);
83: if( pOld==pPtr ){
84: sqlite3_result_error(context, "out of memory", -1);
85: return;
86: }
87: }else{
88: pPtr = sqlite3Fts2HashFind(pHash, zName, nName);
89: if( !pPtr ){
90: char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
91: sqlite3_result_error(context, zErr, -1);
92: sqlite3_free(zErr);
93: return;
94: }
95: }
96:
97: sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
98: }
99:
100: #ifdef SQLITE_TEST
101:
102: #include <tcl.h>
103: #include <string.h>
104:
105: /*
106: ** Implementation of a special SQL scalar function for testing tokenizers
107: ** designed to be used in concert with the Tcl testing framework. This
108: ** function must be called with two arguments:
109: **
110: ** SELECT <function-name>(<key-name>, <input-string>);
111: ** SELECT <function-name>(<key-name>, <pointer>);
112: **
113: ** where <function-name> is the name passed as the second argument
114: ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer')
115: ** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test').
116: **
117: ** The return value is a string that may be interpreted as a Tcl
118: ** list. For each token in the <input-string>, three elements are
119: ** added to the returned list. The first is the token position, the
120: ** second is the token text (folded, stemmed, etc.) and the third is the
121: ** substring of <input-string> associated with the token. For example,
122: ** using the built-in "simple" tokenizer:
123: **
124: ** SELECT fts_tokenizer_test('simple', 'I don't see how');
125: **
126: ** will return the string:
127: **
128: ** "{0 i I 1 dont don't 2 see see 3 how how}"
129: **
130: */
131: static void testFunc(
132: sqlite3_context *context,
133: int argc,
134: sqlite3_value **argv
135: ){
136: fts2Hash *pHash;
137: sqlite3_tokenizer_module *p;
138: sqlite3_tokenizer *pTokenizer = 0;
139: sqlite3_tokenizer_cursor *pCsr = 0;
140:
141: const char *zErr = 0;
142:
143: const char *zName;
144: int nName;
145: const char *zInput;
146: int nInput;
147:
148: const char *zArg = 0;
149:
150: const char *zToken;
151: int nToken;
152: int iStart;
153: int iEnd;
154: int iPos;
155:
156: Tcl_Obj *pRet;
157:
158: assert( argc==2 || argc==3 );
159:
160: nName = sqlite3_value_bytes(argv[0]);
161: zName = (const char *)sqlite3_value_text(argv[0]);
162: nInput = sqlite3_value_bytes(argv[argc-1]);
163: zInput = (const char *)sqlite3_value_text(argv[argc-1]);
164:
165: if( argc==3 ){
166: zArg = (const char *)sqlite3_value_text(argv[1]);
167: }
168:
169: pHash = (fts2Hash *)sqlite3_user_data(context);
170: p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1);
171:
172: if( !p ){
173: char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
174: sqlite3_result_error(context, zErr, -1);
175: sqlite3_free(zErr);
176: return;
177: }
178:
179: pRet = Tcl_NewObj();
180: Tcl_IncrRefCount(pRet);
181:
182: if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
183: zErr = "error in xCreate()";
184: goto finish;
185: }
186: pTokenizer->pModule = p;
187: if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
188: zErr = "error in xOpen()";
189: goto finish;
190: }
191: pCsr->pTokenizer = pTokenizer;
192:
193: while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
194: Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
195: Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
196: zToken = &zInput[iStart];
197: nToken = iEnd-iStart;
198: Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
199: }
200:
201: if( SQLITE_OK!=p->xClose(pCsr) ){
202: zErr = "error in xClose()";
203: goto finish;
204: }
205: if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
206: zErr = "error in xDestroy()";
207: goto finish;
208: }
209:
210: finish:
211: if( zErr ){
212: sqlite3_result_error(context, zErr, -1);
213: }else{
214: sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
215: }
216: Tcl_DecrRefCount(pRet);
217: }
218:
219: static
220: int registerTokenizer(
221: sqlite3 *db,
222: char *zName,
223: const sqlite3_tokenizer_module *p
224: ){
225: int rc;
226: sqlite3_stmt *pStmt;
227: const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
228:
229: rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
230: if( rc!=SQLITE_OK ){
231: return rc;
232: }
233:
234: sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
235: sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
236: sqlite3_step(pStmt);
237:
238: return sqlite3_finalize(pStmt);
239: }
240:
241: static
242: int queryFts2Tokenizer(
243: sqlite3 *db,
244: char *zName,
245: const sqlite3_tokenizer_module **pp
246: ){
247: int rc;
248: sqlite3_stmt *pStmt;
249: const char zSql[] = "SELECT fts2_tokenizer(?)";
250:
251: *pp = 0;
252: rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
253: if( rc!=SQLITE_OK ){
254: return rc;
255: }
256:
257: sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
258: if( SQLITE_ROW==sqlite3_step(pStmt) ){
259: if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
260: memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
261: }
262: }
263:
264: return sqlite3_finalize(pStmt);
265: }
266:
267: void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
268:
269: /*
270: ** Implementation of the scalar function fts2_tokenizer_internal_test().
271: ** This function is used for testing only, it is not included in the
272: ** build unless SQLITE_TEST is defined.
273: **
274: ** The purpose of this is to test that the fts2_tokenizer() function
275: ** can be used as designed by the C-code in the queryFts2Tokenizer and
276: ** registerTokenizer() functions above. These two functions are repeated
277: ** in the README.tokenizer file as an example, so it is important to
278: ** test them.
279: **
280: ** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar
281: ** function with no arguments. An assert() will fail if a problem is
282: ** detected. i.e.:
283: **
284: ** SELECT fts2_tokenizer_internal_test();
285: **
286: */
287: static void intTestFunc(
288: sqlite3_context *context,
289: int argc,
290: sqlite3_value **argv
291: ){
292: int rc;
293: const sqlite3_tokenizer_module *p1;
294: const sqlite3_tokenizer_module *p2;
295: sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
296:
297: /* Test the query function */
298: sqlite3Fts2SimpleTokenizerModule(&p1);
299: rc = queryFts2Tokenizer(db, "simple", &p2);
300: assert( rc==SQLITE_OK );
301: assert( p1==p2 );
302: rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
303: assert( rc==SQLITE_ERROR );
304: assert( p2==0 );
305: assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
306:
307: /* Test the storage function */
308: rc = registerTokenizer(db, "nosuchtokenizer", p1);
309: assert( rc==SQLITE_OK );
310: rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
311: assert( rc==SQLITE_OK );
312: assert( p2==p1 );
313:
314: sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
315: }
316:
317: #endif
318:
319: /*
320: ** Set up SQL objects in database db used to access the contents of
321: ** the hash table pointed to by argument pHash. The hash table must
322: ** been initialised to use string keys, and to take a private copy
323: ** of the key when a value is inserted. i.e. by a call similar to:
324: **
325: ** sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);
326: **
327: ** This function adds a scalar function (see header comment above
328: ** scalarFunc() in this file for details) and, if ENABLE_TABLE is
329: ** defined at compilation time, a temporary virtual table (see header
330: ** comment above struct HashTableVtab) to the database schema. Both
331: ** provide read/write access to the contents of *pHash.
332: **
333: ** The third argument to this function, zName, is used as the name
334: ** of both the scalar and, if created, the virtual table.
335: */
336: int sqlite3Fts2InitHashTable(
337: sqlite3 *db,
338: fts2Hash *pHash,
339: const char *zName
340: ){
341: int rc = SQLITE_OK;
342: void *p = (void *)pHash;
343: const int any = SQLITE_ANY;
344: char *zTest = 0;
345: char *zTest2 = 0;
346:
347: #ifdef SQLITE_TEST
348: void *pdb = (void *)db;
349: zTest = sqlite3_mprintf("%s_test", zName);
350: zTest2 = sqlite3_mprintf("%s_internal_test", zName);
351: if( !zTest || !zTest2 ){
352: rc = SQLITE_NOMEM;
353: }
354: #endif
355:
356: if( rc!=SQLITE_OK
357: || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
358: || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
359: #ifdef SQLITE_TEST
360: || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
361: || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
362: || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
363: #endif
364: );
365:
366: sqlite3_free(zTest);
367: sqlite3_free(zTest2);
368: return rc;
369: }
370:
371: #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts2_tokenizer.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts2