Return to tokenizer.h CVS log

Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts1

sqlite3

    1: /*
    2: ** 2006 July 10
    3: **
    4: ** The author disclaims copyright to this source code.
    5: **
    6: *************************************************************************
    7: ** Defines the interface to tokenizers used by fulltext-search.  There
    8: ** are three basic components:
    9: **
   10: ** sqlite3_tokenizer_module is a singleton defining the tokenizer
   11: ** interface functions.  This is essentially the class structure for
   12: ** tokenizers.
   13: **
   14: ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
   15: ** including customization information defined at creation time.
   16: **
   17: ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
   18: ** tokens from a particular input.
   19: */
   20: #ifndef _TOKENIZER_H_
   21: #define _TOKENIZER_H_
   22: 
   23: /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
   24: ** If tokenizers are to be allowed to call sqlite3_*() functions, then
   25: ** we will need a way to register the API consistently.
   26: */
   27: #include "sqlite3.h"
   28: 
   29: /*
   30: ** Structures used by the tokenizer interface.
   31: */
   32: typedef struct sqlite3_tokenizer sqlite3_tokenizer;
   33: typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
   34: typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
   35: 
   36: struct sqlite3_tokenizer_module {
   37:   int iVersion;                  /* currently 0 */
   38: 
   39:   /*
   40:   ** Create and destroy a tokenizer.  argc/argv are passed down from
   41:   ** the fulltext virtual table creation to allow customization.
   42:   */
   43:   int (*xCreate)(int argc, const char **argv,
   44:                  sqlite3_tokenizer **ppTokenizer);
   45:   int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
   46: 
   47:   /*
   48:   ** Tokenize a particular input.  Call xOpen() to prepare to
   49:   ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
   50:   ** xClose() to free any internal state.  The pInput passed to
   51:   ** xOpen() must exist until the cursor is closed.  The ppToken
   52:   ** result from xNext() is only valid until the next call to xNext()
   53:   ** or until xClose() is called.
   54:   */
   55:   /* TODO(shess) current implementation requires pInput to be
   56:   ** nul-terminated.  This should either be fixed, or pInput/nBytes
   57:   ** should be converted to zInput.
   58:   */
   59:   int (*xOpen)(sqlite3_tokenizer *pTokenizer,
   60:                const char *pInput, int nBytes,
   61:                sqlite3_tokenizer_cursor **ppCursor);
   62:   int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
   63:   int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
   64:                const char **ppToken, int *pnBytes,
   65:                int *piStartOffset, int *piEndOffset, int *piPosition);
   66: };
   67: 
   68: struct sqlite3_tokenizer {
   69:   sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
   70:   /* Tokenizer implementations will typically add additional fields */
   71: };
   72: 
   73: struct sqlite3_tokenizer_cursor {
   74:   sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
   75:   /* Tokenizer implementations will typically add additional fields */
   76: };
   77: 
   78: /*
   79: ** Get the module for a tokenizer which generates tokens based on a
   80: ** set of non-token characters.  The default is to break tokens at any
   81: ** non-alnum character, though the set of delimiters can also be
   82: ** specified by the first argv argument to xCreate().
   83: */
   84: /* TODO(shess) This doesn't belong here.  Need some sort of
   85: ** registration process.
   86: */
   87: void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule);
   88: 
   89: #endif /* _TOKENIZER_H_ */