Return to tokenizer.h CVS log

Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts1

Annotation of embedaddon/sqlite3/ext/fts1/tokenizer.h, revision 1.1.1.1

1.1       misho       1: /*
                      2: ** 2006 July 10
                      3: **
                      4: ** The author disclaims copyright to this source code.
                      5: **
                      6: *************************************************************************
                      7: ** Defines the interface to tokenizers used by fulltext-search.  There
                      8: ** are three basic components:
                      9: **
                     10: ** sqlite3_tokenizer_module is a singleton defining the tokenizer
                     11: ** interface functions.  This is essentially the class structure for
                     12: ** tokenizers.
                     13: **
                     14: ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
                     15: ** including customization information defined at creation time.
                     16: **
                     17: ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
                     18: ** tokens from a particular input.
                     19: */
                     20: #ifndef _TOKENIZER_H_
                     21: #define _TOKENIZER_H_
                     22: 
                     23: /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
                     24: ** If tokenizers are to be allowed to call sqlite3_*() functions, then
                     25: ** we will need a way to register the API consistently.
                     26: */
                     27: #include "sqlite3.h"
                     28: 
                     29: /*
                     30: ** Structures used by the tokenizer interface.
                     31: */
                     32: typedef struct sqlite3_tokenizer sqlite3_tokenizer;
                     33: typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
                     34: typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
                     35: 
                     36: struct sqlite3_tokenizer_module {
                     37:   int iVersion;                  /* currently 0 */
                     38: 
                     39:   /*
                     40:   ** Create and destroy a tokenizer.  argc/argv are passed down from
                     41:   ** the fulltext virtual table creation to allow customization.
                     42:   */
                     43:   int (*xCreate)(int argc, const char **argv,
                     44:                  sqlite3_tokenizer **ppTokenizer);
                     45:   int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
                     46: 
                     47:   /*
                     48:   ** Tokenize a particular input.  Call xOpen() to prepare to
                     49:   ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
                     50:   ** xClose() to free any internal state.  The pInput passed to
                     51:   ** xOpen() must exist until the cursor is closed.  The ppToken
                     52:   ** result from xNext() is only valid until the next call to xNext()
                     53:   ** or until xClose() is called.
                     54:   */
                     55:   /* TODO(shess) current implementation requires pInput to be
                     56:   ** nul-terminated.  This should either be fixed, or pInput/nBytes
                     57:   ** should be converted to zInput.
                     58:   */
                     59:   int (*xOpen)(sqlite3_tokenizer *pTokenizer,
                     60:                const char *pInput, int nBytes,
                     61:                sqlite3_tokenizer_cursor **ppCursor);
                     62:   int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
                     63:   int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
                     64:                const char **ppToken, int *pnBytes,
                     65:                int *piStartOffset, int *piEndOffset, int *piPosition);
                     66: };
                     67: 
                     68: struct sqlite3_tokenizer {
                     69:   sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
                     70:   /* Tokenizer implementations will typically add additional fields */
                     71: };
                     72: 
                     73: struct sqlite3_tokenizer_cursor {
                     74:   sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
                     75:   /* Tokenizer implementations will typically add additional fields */
                     76: };
                     77: 
                     78: /*
                     79: ** Get the module for a tokenizer which generates tokens based on a
                     80: ** set of non-token characters.  The default is to break tokens at any
                     81: ** non-alnum character, though the set of delimiters can also be
                     82: ** specified by the first argv argument to xCreate().
                     83: */
                     84: /* TODO(shess) This doesn't belong here.  Need some sort of
                     85: ** registration process.
                     86: */
                     87: void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule);
                     88: 
                     89: #endif /* _TOKENIZER_H_ */