Annotation of embedaddon/sqlite3/ext/fts2/fts2_icu.c, revision 1.1
1.1 ! misho 1: /*
! 2: ** 2007 June 22
! 3: **
! 4: ** The author disclaims copyright to this source code. In place of
! 5: ** a legal notice, here is a blessing:
! 6: **
! 7: ** May you do good and not evil.
! 8: ** May you find forgiveness for yourself and forgive others.
! 9: ** May you share freely, never taking more than you give.
! 10: **
! 11: *************************************************************************
! 12: ** This file implements a tokenizer for fts2 based on the ICU library.
! 13: **
! 14: ** $Id: fts2_icu.c,v 1.3 2008/12/18 05:30:26 danielk1977 Exp $
! 15: */
! 16:
! 17: #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
! 18: #ifdef SQLITE_ENABLE_ICU
! 19:
! 20: #include <assert.h>
! 21: #include <string.h>
! 22: #include "fts2_tokenizer.h"
! 23:
! 24: #include <unicode/ubrk.h>
! 25: #include <unicode/ucol.h>
! 26: #include <unicode/ustring.h>
! 27: #include <unicode/utf16.h>
! 28:
! 29: typedef struct IcuTokenizer IcuTokenizer;
! 30: typedef struct IcuCursor IcuCursor;
! 31:
! 32: struct IcuTokenizer {
! 33: sqlite3_tokenizer base;
! 34: char *zLocale;
! 35: };
! 36:
! 37: struct IcuCursor {
! 38: sqlite3_tokenizer_cursor base;
! 39:
! 40: UBreakIterator *pIter; /* ICU break-iterator object */
! 41: int nChar; /* Number of UChar elements in pInput */
! 42: UChar *aChar; /* Copy of input using utf-16 encoding */
! 43: int *aOffset; /* Offsets of each character in utf-8 input */
! 44:
! 45: int nBuffer;
! 46: char *zBuffer;
! 47:
! 48: int iToken;
! 49: };
! 50:
! 51: /*
! 52: ** Create a new tokenizer instance.
! 53: */
! 54: static int icuCreate(
! 55: int argc, /* Number of entries in argv[] */
! 56: const char * const *argv, /* Tokenizer creation arguments */
! 57: sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
! 58: ){
! 59: IcuTokenizer *p;
! 60: int n = 0;
! 61:
! 62: if( argc>0 ){
! 63: n = strlen(argv[0])+1;
! 64: }
! 65: p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
! 66: if( !p ){
! 67: return SQLITE_NOMEM;
! 68: }
! 69: memset(p, 0, sizeof(IcuTokenizer));
! 70:
! 71: if( n ){
! 72: p->zLocale = (char *)&p[1];
! 73: memcpy(p->zLocale, argv[0], n);
! 74: }
! 75:
! 76: *ppTokenizer = (sqlite3_tokenizer *)p;
! 77:
! 78: return SQLITE_OK;
! 79: }
! 80:
! 81: /*
! 82: ** Destroy a tokenizer
! 83: */
! 84: static int icuDestroy(sqlite3_tokenizer *pTokenizer){
! 85: IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
! 86: sqlite3_free(p);
! 87: return SQLITE_OK;
! 88: }
! 89:
! 90: /*
! 91: ** Prepare to begin tokenizing a particular string. The input
! 92: ** string to be tokenized is pInput[0..nBytes-1]. A cursor
! 93: ** used to incrementally tokenize this string is returned in
! 94: ** *ppCursor.
! 95: */
! 96: static int icuOpen(
! 97: sqlite3_tokenizer *pTokenizer, /* The tokenizer */
! 98: const char *zInput, /* Input string */
! 99: int nInput, /* Length of zInput in bytes */
! 100: sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
! 101: ){
! 102: IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
! 103: IcuCursor *pCsr;
! 104:
! 105: const int32_t opt = U_FOLD_CASE_DEFAULT;
! 106: UErrorCode status = U_ZERO_ERROR;
! 107: int nChar;
! 108:
! 109: UChar32 c;
! 110: int iInput = 0;
! 111: int iOut = 0;
! 112:
! 113: *ppCursor = 0;
! 114:
! 115: if( nInput<0 ){
! 116: nInput = strlen(zInput);
! 117: }
! 118: nChar = nInput+1;
! 119: pCsr = (IcuCursor *)sqlite3_malloc(
! 120: sizeof(IcuCursor) + /* IcuCursor */
! 121: nChar * sizeof(UChar) + /* IcuCursor.aChar[] */
! 122: (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */
! 123: );
! 124: if( !pCsr ){
! 125: return SQLITE_NOMEM;
! 126: }
! 127: memset(pCsr, 0, sizeof(IcuCursor));
! 128: pCsr->aChar = (UChar *)&pCsr[1];
! 129: pCsr->aOffset = (int *)&pCsr->aChar[nChar];
! 130:
! 131: pCsr->aOffset[iOut] = iInput;
! 132: U8_NEXT(zInput, iInput, nInput, c);
! 133: while( c>0 ){
! 134: int isError = 0;
! 135: c = u_foldCase(c, opt);
! 136: U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
! 137: if( isError ){
! 138: sqlite3_free(pCsr);
! 139: return SQLITE_ERROR;
! 140: }
! 141: pCsr->aOffset[iOut] = iInput;
! 142:
! 143: if( iInput<nInput ){
! 144: U8_NEXT(zInput, iInput, nInput, c);
! 145: }else{
! 146: c = 0;
! 147: }
! 148: }
! 149:
! 150: pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
! 151: if( !U_SUCCESS(status) ){
! 152: sqlite3_free(pCsr);
! 153: return SQLITE_ERROR;
! 154: }
! 155: pCsr->nChar = iOut;
! 156:
! 157: ubrk_first(pCsr->pIter);
! 158: *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
! 159: return SQLITE_OK;
! 160: }
! 161:
! 162: /*
! 163: ** Close a tokenization cursor previously opened by a call to icuOpen().
! 164: */
! 165: static int icuClose(sqlite3_tokenizer_cursor *pCursor){
! 166: IcuCursor *pCsr = (IcuCursor *)pCursor;
! 167: ubrk_close(pCsr->pIter);
! 168: sqlite3_free(pCsr->zBuffer);
! 169: sqlite3_free(pCsr);
! 170: return SQLITE_OK;
! 171: }
! 172:
! 173: /*
! 174: ** Extract the next token from a tokenization cursor.
! 175: */
! 176: static int icuNext(
! 177: sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
! 178: const char **ppToken, /* OUT: *ppToken is the token text */
! 179: int *pnBytes, /* OUT: Number of bytes in token */
! 180: int *piStartOffset, /* OUT: Starting offset of token */
! 181: int *piEndOffset, /* OUT: Ending offset of token */
! 182: int *piPosition /* OUT: Position integer of token */
! 183: ){
! 184: IcuCursor *pCsr = (IcuCursor *)pCursor;
! 185:
! 186: int iStart = 0;
! 187: int iEnd = 0;
! 188: int nByte = 0;
! 189:
! 190: while( iStart==iEnd ){
! 191: UChar32 c;
! 192:
! 193: iStart = ubrk_current(pCsr->pIter);
! 194: iEnd = ubrk_next(pCsr->pIter);
! 195: if( iEnd==UBRK_DONE ){
! 196: return SQLITE_DONE;
! 197: }
! 198:
! 199: while( iStart<iEnd ){
! 200: int iWhite = iStart;
! 201: U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
! 202: if( u_isspace(c) ){
! 203: iStart = iWhite;
! 204: }else{
! 205: break;
! 206: }
! 207: }
! 208: assert(iStart<=iEnd);
! 209: }
! 210:
! 211: do {
! 212: UErrorCode status = U_ZERO_ERROR;
! 213: if( nByte ){
! 214: char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
! 215: if( !zNew ){
! 216: return SQLITE_NOMEM;
! 217: }
! 218: pCsr->zBuffer = zNew;
! 219: pCsr->nBuffer = nByte;
! 220: }
! 221:
! 222: u_strToUTF8(
! 223: pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */
! 224: &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */
! 225: &status /* Output success/failure */
! 226: );
! 227: } while( nByte>pCsr->nBuffer );
! 228:
! 229: *ppToken = pCsr->zBuffer;
! 230: *pnBytes = nByte;
! 231: *piStartOffset = pCsr->aOffset[iStart];
! 232: *piEndOffset = pCsr->aOffset[iEnd];
! 233: *piPosition = pCsr->iToken++;
! 234:
! 235: return SQLITE_OK;
! 236: }
! 237:
! 238: /*
! 239: ** The set of routines that implement the simple tokenizer
! 240: */
! 241: static const sqlite3_tokenizer_module icuTokenizerModule = {
! 242: 0, /* iVersion */
! 243: icuCreate, /* xCreate */
! 244: icuDestroy, /* xCreate */
! 245: icuOpen, /* xOpen */
! 246: icuClose, /* xClose */
! 247: icuNext, /* xNext */
! 248: };
! 249:
! 250: /*
! 251: ** Set *ppModule to point at the implementation of the ICU tokenizer.
! 252: */
! 253: void sqlite3Fts2IcuTokenizerModule(
! 254: sqlite3_tokenizer_module const**ppModule
! 255: ){
! 256: *ppModule = &icuTokenizerModule;
! 257: }
! 258:
! 259: #endif /* defined(SQLITE_ENABLE_ICU) */
! 260: #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts2_icu.c CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / ext / fts2