Return to simple_tokenizer.c CVS log

Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts1

Annotation of embedaddon/sqlite3/ext/fts1/simple_tokenizer.c, revision 1.1.1.1

1.1       misho       1: /*
                      2: ** The author disclaims copyright to this source code.
                      3: **
                      4: *************************************************************************
                      5: ** Implementation of the "simple" full-text-search tokenizer.
                      6: */
                      7: 
                      8: #include <assert.h>
                      9: #if !defined(__APPLE__)
                     10: #include <malloc.h>
                     11: #else
                     12: #include <stdlib.h>
                     13: #endif
                     14: #include <stdio.h>
                     15: #include <string.h>
                     16: #include <ctype.h>
                     17: 
                     18: #include "tokenizer.h"
                     19: 
                     20: /* Duplicate a string; the caller must free() the returned string.
                     21:  * (We don't use strdup() since it's not part of the standard C library and
                     22:  * may not be available everywhere.) */
                     23: /* TODO(shess) Copied from fulltext.c, consider util.c for such
                     24: ** things. */
                     25: static char *string_dup(const char *s){
                     26:   char *str = malloc(strlen(s) + 1);
                     27:   strcpy(str, s);
                     28:   return str;
                     29: }
                     30: 
                     31: typedef struct simple_tokenizer {
                     32:   sqlite3_tokenizer base;
                     33:   const char *zDelim;          /* token delimiters */
                     34: } simple_tokenizer;
                     35: 
                     36: typedef struct simple_tokenizer_cursor {
                     37:   sqlite3_tokenizer_cursor base;
                     38:   const char *pInput;          /* input we are tokenizing */
                     39:   int nBytes;                  /* size of the input */
                     40:   const char *pCurrent;        /* current position in pInput */
                     41:   int iToken;                  /* index of next token to be returned */
                     42:   char *zToken;                /* storage for current token */
                     43:   int nTokenBytes;             /* actual size of current token */
                     44:   int nTokenAllocated;         /* space allocated to zToken buffer */
                     45: } simple_tokenizer_cursor;
                     46: 
                     47: static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */
                     48: 
                     49: static int simpleCreate(
                     50:   int argc, const char **argv,
                     51:   sqlite3_tokenizer **ppTokenizer
                     52: ){
                     53:   simple_tokenizer *t;
                     54: 
                     55:   t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer));
                     56:   /* TODO(shess) Delimiters need to remain the same from run to run,
                     57:   ** else we need to reindex.  One solution would be a meta-table to
                     58:   ** track such information in the database, then we'd only want this
                     59:   ** information on the initial create.
                     60:   */
                     61:   if( argc>1 ){
                     62:     t->zDelim = string_dup(argv[1]);
                     63:   } else {
                     64:     /* Build a string excluding alphanumeric ASCII characters */
                     65:     char zDelim[0x80];               /* nul-terminated, so nul not a member */
                     66:     int i, j;
                     67:     for(i=1, j=0; i<0x80; i++){
                     68:       if( !isalnum(i) ){
                     69:         zDelim[j++] = i;
                     70:       }
                     71:     }
                     72:     zDelim[j++] = '\0';
                     73:     assert( j<=sizeof(zDelim) );
                     74:     t->zDelim = string_dup(zDelim);
                     75:   }
                     76: 
                     77:   *ppTokenizer = &t->base;
                     78:   return SQLITE_OK;
                     79: }
                     80: 
                     81: static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
                     82:   simple_tokenizer *t = (simple_tokenizer *) pTokenizer;
                     83: 
                     84:   free((void *) t->zDelim);
                     85:   free(t);
                     86: 
                     87:   return SQLITE_OK;
                     88: }
                     89: 
                     90: static int simpleOpen(
                     91:   sqlite3_tokenizer *pTokenizer,
                     92:   const char *pInput, int nBytes,
                     93:   sqlite3_tokenizer_cursor **ppCursor
                     94: ){
                     95:   simple_tokenizer_cursor *c;
                     96: 
                     97:   c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
                     98:   c->pInput = pInput;
                     99:   c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes;
                    100:   c->pCurrent = c->pInput;        /* start tokenizing at the beginning */
                    101:   c->iToken = 0;
                    102:   c->zToken = NULL;               /* no space allocated, yet. */
                    103:   c->nTokenBytes = 0;
                    104:   c->nTokenAllocated = 0;
                    105: 
                    106:   *ppCursor = &c->base;
                    107:   return SQLITE_OK;
                    108: }
                    109: 
                    110: static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
                    111:   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
                    112: 
                    113:   if( NULL!=c->zToken ){
                    114:     free(c->zToken);
                    115:   }
                    116:   free(c);
                    117: 
                    118:   return SQLITE_OK;
                    119: }
                    120: 
                    121: static int simpleNext(
                    122:   sqlite3_tokenizer_cursor *pCursor,
                    123:   const char **ppToken, int *pnBytes,
                    124:   int *piStartOffset, int *piEndOffset, int *piPosition
                    125: ){
                    126:   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
                    127:   simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
                    128:   int ii;
                    129: 
                    130:   while( c->pCurrent-c->pInput<c->nBytes ){
                    131:     int n = (int) strcspn(c->pCurrent, t->zDelim);
                    132:     if( n>0 ){
                    133:       if( n+1>c->nTokenAllocated ){
                    134:         c->zToken = realloc(c->zToken, n+1);
                    135:       }
                    136:       for(ii=0; ii<n; ii++){
                    137:         /* TODO(shess) This needs expansion to handle UTF-8
                    138:         ** case-insensitivity.
                    139:         */
                    140:         char ch = c->pCurrent[ii];
                    141:         c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch;
                    142:       }
                    143:       c->zToken[n] = '\0';
                    144:       *ppToken = c->zToken;
                    145:       *pnBytes = n;
                    146:       *piStartOffset = (int) (c->pCurrent-c->pInput);
                    147:       *piEndOffset = *piStartOffset+n;
                    148:       *piPosition = c->iToken++;
                    149:       c->pCurrent += n + 1;
                    150: 
                    151:       return SQLITE_OK;
                    152:     }
                    153:     c->pCurrent += n + 1;
                    154:     /* TODO(shess) could strspn() to skip delimiters en masse.  Needs
                    155:     ** to happen in two places, though, which is annoying.
                    156:     */
                    157:   }
                    158:   return SQLITE_DONE;
                    159: }
                    160: 
                    161: static sqlite3_tokenizer_module simpleTokenizerModule = {
                    162:   0,
                    163:   simpleCreate,
                    164:   simpleDestroy,
                    165:   simpleOpen,
                    166:   simpleClose,
                    167:   simpleNext,
                    168: };
                    169: 
                    170: void get_simple_tokenizer_module(
                    171:   sqlite3_tokenizer_module **ppModule
                    172: ){
                    173:   *ppModule = &simpleTokenizerModule;
                    174: }