Annotation of embedaddon/sqlite3/test/fts3cov.test, revision 1.1
1.1 ! misho 1: # 2009 December 03
! 2: #
! 3: # May you do good and not evil.
! 4: # May you find forgiveness for yourself and forgive others.
! 5: # May you share freely, never taking more than you give.
! 6: #
! 7: #***********************************************************************
! 8: #
! 9: # The tests in this file are structural coverage tests for FTS3.
! 10: #
! 11:
! 12: set testdir [file dirname $argv0]
! 13: source $testdir/tester.tcl
! 14:
! 15: # If this build does not include FTS3, skip the tests in this file.
! 16: #
! 17: ifcapable !fts3 { finish_test ; return }
! 18: source $testdir/fts3_common.tcl
! 19: source $testdir/malloc_common.tcl
! 20:
! 21: set DO_MALLOC_TEST 0
! 22: set testprefix fts3cov
! 23:
! 24: #--------------------------------------------------------------------------
! 25: # When it first needs to read a block from the %_segments table, the FTS3
! 26: # module compiles an SQL statement for that purpose. The statement is
! 27: # stored and reused each subsequent time a block is read. This test case
! 28: # tests the effects of an OOM error occuring while compiling the statement.
! 29: #
! 30: # Similarly, when FTS3 first needs to scan through a set of segment leaves
! 31: # to find a set of documents that matches a term, it allocates a string
! 32: # containing the text of the required SQL, and compiles one or more
! 33: # statements to traverse the leaves. This test case tests that OOM errors
! 34: # that occur while allocating this string and statement are handled correctly
! 35: # also.
! 36: #
! 37: do_test fts3cov-1.1 {
! 38: execsql {
! 39: CREATE VIRTUAL TABLE t1 USING fts3(x);
! 40: INSERT INTO t1(t1) VALUES('nodesize=24');
! 41: BEGIN;
! 42: INSERT INTO t1 VALUES('Is the night chilly and dark?');
! 43: INSERT INTO t1 VALUES('The night is chilly, but not dark.');
! 44: INSERT INTO t1 VALUES('The thin gray cloud is spread on high,');
! 45: INSERT INTO t1 VALUES('It covers but not hides the sky.');
! 46: COMMIT;
! 47: SELECT count(*)>0 FROM t1_segments;
! 48: }
! 49: } {1}
! 50:
! 51: set DO_MALLOC_TEST 1
! 52: do_restart_select_test fts3cov-1.2 {
! 53: SELECT docid FROM t1 WHERE t1 MATCH 'chilly';
! 54: } {1 2}
! 55: set DO_MALLOC_TEST 0
! 56:
! 57: #--------------------------------------------------------------------------
! 58: # When querying the full-text index, if an expected internal node block is
! 59: # missing from the %_segments table, or if a NULL value is stored in the
! 60: # %_segments table instead of a binary blob, database corruption should be
! 61: # reported.
! 62: #
! 63: # Even with tiny 24 byte nodes, it takes a fair bit of data to produce a
! 64: # segment b-tree that uses the %_segments table to store internal nodes.
! 65: #
! 66: do_test fts3cov-2.1 {
! 67: execsql {
! 68: INSERT INTO t1(t1) VALUES('nodesize=24');
! 69: BEGIN;
! 70: INSERT INTO t1 VALUES('The moon is behind, and at the full;');
! 71: INSERT INTO t1 VALUES('And yet she looks both small and dull.');
! 72: INSERT INTO t1 VALUES('The night is chill, the cloud is gray:');
! 73: INSERT INTO t1 VALUES('''T is a month before the month of May,');
! 74: INSERT INTO t1 VALUES('And the Spring comes slowly up this way.');
! 75: INSERT INTO t1 VALUES('The lovely lady, Christabel,');
! 76: INSERT INTO t1 VALUES('Whom her father loves so well,');
! 77: INSERT INTO t1 VALUES('What makes her in the wood so late,');
! 78: INSERT INTO t1 VALUES('A furlong from the castle gate?');
! 79: INSERT INTO t1 VALUES('She had dreams all yesternight');
! 80: INSERT INTO t1 VALUES('Of her own betrothed knight;');
! 81: INSERT INTO t1 VALUES('And she in the midnight wood will pray');
! 82: INSERT INTO t1 VALUES('For the weal of her lover that''s far away.');
! 83: COMMIT;
! 84: }
! 85: execsql {
! 86: INSERT INTO t1(t1) VALUES('optimize');
! 87: SELECT substr(hex(root), 1, 2) FROM t1_segdir;
! 88: }
! 89: } {03}
! 90:
! 91: # Test the "missing entry" case:
! 92: do_test fts3cov-2.2 {
! 93: set root [db one {SELECT root FROM t1_segdir}]
! 94: read_fts3varint [string range $root 1 end] left_child
! 95: execsql { DELETE FROM t1_segments WHERE blockid = $left_child }
! 96: } {}
! 97: do_error_test fts3cov-2.3 {
! 98: SELECT * FROM t1 WHERE t1 MATCH 'c*'
! 99: } {SQL logic error or missing database}
! 100:
! 101: # Test the "replaced with NULL" case:
! 102: do_test fts3cov-2.4 {
! 103: execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }
! 104: } {}
! 105: do_error_test fts3cov-2.5 {
! 106: SELECT * FROM t1 WHERE t1 MATCH 'cloud'
! 107: } {SQL logic error or missing database}
! 108:
! 109: #--------------------------------------------------------------------------
! 110: # The following tests are to test the effects of OOM errors while storing
! 111: # terms in the pending-hash table. Specifically, while creating doclist
! 112: # blobs to store in the table. More specifically, to test OOM errors while
! 113: # appending column numbers to doclists. For example, if a doclist consists
! 114: # of:
! 115: #
! 116: # <docid> <column 0 offset-list> 0x01 <column N> <column N offset-list>
! 117: #
! 118: # The following tests check that malloc errors encountered while appending
! 119: # the "0x01 <column N>" data to the dynamically growable blob used to
! 120: # accumulate the doclist in memory are handled correctly.
! 121: #
! 122: do_test fts3cov-3.1 {
! 123: set cols [list]
! 124: set vals [list]
! 125: for {set i 0} {$i < 120} {incr i} {
! 126: lappend cols "col$i"
! 127: lappend vals "'word'"
! 128: }
! 129: execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"
! 130: } {}
! 131: set DO_MALLOC_TEST 1
! 132: do_write_test fts3cov-3.2 t2_content "
! 133: INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,])
! 134: "
! 135: do_write_test fts3cov-3.3 t2_content "
! 136: INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,])
! 137: "
! 138: do_write_test fts3cov-3.4 t2_content "
! 139: INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,])
! 140: "
! 141:
! 142: #-------------------------------------------------------------------------
! 143: # If too much data accumulates in the pending-terms hash table, it is
! 144: # flushed to the database automatically, even if the transaction has not
! 145: # finished. The following tests check the effects of encountering an OOM
! 146: # while doing this.
! 147: #
! 148: do_test fts3cov-4.1 {
! 149: execsql {
! 150: CREATE VIRTUAL TABLE t3 USING fts3(x);
! 151: INSERT INTO t3(t3) VALUES('nodesize=24');
! 152: INSERT INTO t3(t3) VALUES('maxpending=100');
! 153: }
! 154: } {}
! 155: set DO_MALLOC_TEST 1
! 156: do_write_test fts3cov-4.2 t3_content {
! 157: INSERT INTO t3(docid, x)
! 158: SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL
! 159: SELECT 3, 'And comforted fair Geraldine:' UNION ALL
! 160: SELECT 4, '''O well, bright dame, may you command' UNION ALL
! 161: SELECT 5, 'The service of Sir Leoline;' UNION ALL
! 162: SELECT 2, 'And gladly our stout chivalry' UNION ALL
! 163: SELECT 7, 'Will he send forth, and friends withal,' UNION ALL
! 164: SELECT 8, 'To guide and guard you safe and free' UNION ALL
! 165: SELECT 6, 'Home to your noble father''s hall.'''
! 166: }
! 167:
! 168: #-------------------------------------------------------------------------
! 169: # When building the internal tree structure for each segment b-tree, FTS3
! 170: # assumes that the content of each internal node will be less than
! 171: # $nodesize bytes, where $nodesize is the advisory node size. If this turns
! 172: # out to be untrue, then an extra buffer must be malloc'd for each term.
! 173: # This test case tests these paths and the effects of said mallocs failing
! 174: # by inserting insert a document with some fairly large terms into a
! 175: # full-text table with a very small node-size.
! 176: #
! 177: # Test this handling of large terms in three contexts:
! 178: #
! 179: # 1. When flushing the pending-terms table.
! 180: # 2. When optimizing the data structures using the INSERT syntax.
! 181: # 2. When optimizing the data structures using the deprecated SELECT syntax.
! 182: #
! 183: do_test fts3cov-5.1 {
! 184: execsql {
! 185: CREATE VIRTUAL TABLE t4 USING fts3(x);
! 186: INSERT INTO t4(t4) VALUES('nodesize=24');
! 187: }
! 188: } {}
! 189: set DO_MALLOC_TEST 1
! 190:
! 191: # Test when flushing pending-terms table.
! 192: do_write_test fts3cov-5.2 t4_content {
! 193: INSERT INTO t4
! 194: SELECT 'ItisanancientMarinerAndhestoppethoneofthreeAA' UNION ALL
! 195: SELECT 'ItisanancientMarinerAndhestoppethoneofthreeBB' UNION ALL
! 196: SELECT 'ItisanancientMarinerAndhestoppethoneofthreeCC' UNION ALL
! 197: SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstAA' UNION ALL
! 198: SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstBB' UNION ALL
! 199: SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstCC'
! 200: }
! 201:
! 202: # Test when optimizing via INSERT.
! 203: do_test fts3cov-5.3 { execsql { INSERT INTO t4 VALUES('extra!') } } {}
! 204: do_write_test fts3cov-5.2 t4_segments { INSERT INTO t4(t4) VALUES('optimize') }
! 205:
! 206: # Test when optimizing via SELECT.
! 207: do_test fts3cov-5.5 { execsql { INSERT INTO t4 VALUES('more extra!') } } {}
! 208: do_write_test fts3cov-5.6 t4_segments {
! 209: SELECT * FROM (SELECT optimize(t4) FROM t4 LIMIT 1)
! 210: EXCEPT SELECT 'Index optimized'
! 211: }
! 212:
! 213: #-------------------------------------------------------------------------
! 214: # When merging all segments at a given level to create a single segment
! 215: # at level+1, FTS3 runs a query of the form:
! 216: #
! 217: # SELECT count(*) FROM %_segdir WHERE level = ?
! 218: #
! 219: # The query is compiled the first time this operation is required and
! 220: # reused thereafter. This test aims to test the effects of an OOM while
! 221: # preparing and executing this query for the first time.
! 222: #
! 223: # Then, keep inserting rows into the table so that the effects of an OOM
! 224: # while re-executing the same query can also be tested.
! 225: #
! 226: do_test fts3cov-6.1 {
! 227: execsql { CREATE VIRTUAL TABLE t5 USING fts3(x) }
! 228: for {set i 0} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
! 229: execsql { SELECT count(*) FROM t5_segdir }
! 230: } {16}
! 231:
! 232: # First time.
! 233: db close
! 234: sqlite3 db test.db
! 235: do_write_test fts3cov-6.2 t5_content {
! 236: INSERT INTO t5 VALUES('segment number 16!');
! 237: }
! 238:
! 239: # Second time.
! 240: do_test fts3cov-6.3 {
! 241: for {set i 1} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
! 242: execsql { SELECT count(*) FROM t5_segdir }
! 243: } {17}
! 244: do_write_test fts3cov-6.4 t5_content {
! 245: INSERT INTO t5 VALUES('segment number 16!');
! 246: }
! 247:
! 248: #-------------------------------------------------------------------------
! 249: # Update the docid of a row. Test this in two scenarios:
! 250: #
! 251: # 1. When the row being updated is the only row in the table.
! 252: # 2. When it is not.
! 253: #
! 254: # The two cases above take different paths because in case 1 all data
! 255: # structures can simply be emptied before inserting the new row record.
! 256: # In case 2, the data structures actually have to be updated.
! 257: #
! 258: do_test fts3cov-7.1 {
! 259: execsql {
! 260: CREATE VIRTUAL TABLE t7 USING fts3(a, b, c);
! 261: INSERT INTO t7 VALUES('A', 'B', 'C');
! 262: UPDATE t7 SET docid = 5;
! 263: SELECT docid, * FROM t7;
! 264: }
! 265: } {5 A B C}
! 266: do_test fts3cov-7.2 {
! 267: execsql {
! 268: INSERT INTO t7 VALUES('D', 'E', 'F');
! 269: UPDATE t7 SET docid = 1 WHERE docid = 6;
! 270: SELECT docid, * FROM t7;
! 271: }
! 272: } {1 D E F 5 A B C}
! 273:
! 274: #-------------------------------------------------------------------------
! 275: # If a set of documents are modified within a transaction, the
! 276: # pending-terms table must be flushed each time a document with a docid
! 277: # less than or equal to the previous docid is modified.
! 278: #
! 279: # This test checks the effects of an OOM error occuring when the
! 280: # pending-terms table is flushed for this reason as part of a DELETE
! 281: # statement.
! 282: #
! 283: do_malloc_test fts3cov-8 -sqlprep {
! 284: BEGIN;
! 285: CREATE VIRTUAL TABLE t8 USING fts3;
! 286: INSERT INTO t8 VALUES('the output of each batch run');
! 287: INSERT INTO t8 VALUES('(possibly a day''s work)');
! 288: INSERT INTO t8 VALUES('was written to two separate disks');
! 289: COMMIT;
! 290: } -sqlbody {
! 291: BEGIN;
! 292: DELETE FROM t8 WHERE rowid = 3;
! 293: DELETE FROM t8 WHERE rowid = 2;
! 294: DELETE FROM t8 WHERE rowid = 1;
! 295: COMMIT;
! 296: }
! 297:
! 298: #-------------------------------------------------------------------------
! 299: # Test some branches in the code that handles "special" inserts like:
! 300: #
! 301: # INSERT INTO t1(t1) VALUES('optimize');
! 302: #
! 303: # Also test that an optimize (INSERT method) works on an empty table.
! 304: #
! 305: set DO_MALLOC_TEST 0
! 306: do_test fts3cov-9.1 {
! 307: execsql { CREATE VIRTUAL TABLE xx USING fts3 }
! 308: } {}
! 309: do_error_test fts3cov-9.2 {
! 310: INSERT INTO xx(xx) VALUES('optimise'); -- British spelling
! 311: } {SQL logic error or missing database}
! 312: do_error_test fts3cov-9.3 {
! 313: INSERT INTO xx(xx) VALUES('short');
! 314: } {SQL logic error or missing database}
! 315: do_error_test fts3cov-9.4 {
! 316: INSERT INTO xx(xx) VALUES('waytoolongtobecorrect');
! 317: } {SQL logic error or missing database}
! 318: do_test fts3cov-9.5 {
! 319: execsql { INSERT INTO xx(xx) VALUES('optimize') }
! 320: } {}
! 321:
! 322: #-------------------------------------------------------------------------
! 323: # Test that a table can be optimized in the middle of a transaction when
! 324: # the pending-terms table is non-empty. This case involves some extra
! 325: # branches because data must be read not only from the database, but
! 326: # also from the pending-terms table.
! 327: #
! 328: do_malloc_test fts3cov-10 -sqlprep {
! 329: CREATE VIRTUAL TABLE t10 USING fts3;
! 330: INSERT INTO t10 VALUES('Optimising images for the web is a tricky business');
! 331: BEGIN;
! 332: INSERT INTO t10 VALUES('You have to get the right balance between');
! 333: } -sqlbody {
! 334: INSERT INTO t10(t10) VALUES('optimize');
! 335: }
! 336:
! 337: #-------------------------------------------------------------------------
! 338: # Test a full-text query for a term that was once in the index, but is
! 339: # no longer.
! 340: #
! 341: do_test fts3cov-11.1 {
! 342: execsql {
! 343: CREATE VIRTUAL TABLE xx USING fts3;
! 344: INSERT INTO xx VALUES('one two three');
! 345: INSERT INTO xx VALUES('four five six');
! 346: DELETE FROM xx WHERE docid = 1;
! 347: }
! 348: execsql { SELECT * FROM xx WHERE xx MATCH 'two' }
! 349: } {}
! 350:
! 351:
! 352: do_malloc_test fts3cov-12 -sqlprep {
! 353: CREATE VIRTUAL TABLE t12 USING fts3;
! 354: INSERT INTO t12 VALUES('is one of the two togther');
! 355: BEGIN;
! 356: INSERT INTO t12 VALUES('one which was appropriate at the time');
! 357: } -sqlbody {
! 358: SELECT * FROM t12 WHERE t12 MATCH 'one'
! 359: }
! 360:
! 361: do_malloc_test fts3cov-13 -sqlprep {
! 362: PRAGMA encoding = 'UTF-16';
! 363: CREATE VIRTUAL TABLE t13 USING fts3;
! 364: INSERT INTO t13 VALUES('two scalar functions');
! 365: INSERT INTO t13 VALUES('scalar two functions');
! 366: INSERT INTO t13 VALUES('functions scalar two');
! 367: } -sqlbody {
! 368: SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two';
! 369: SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two';
! 370: SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two';
! 371: }
! 372:
! 373: do_execsql_test 14.0 {
! 374: CREATE VIRTUAL TABLE t14 USING fts4(a, b);
! 375: INSERT INTO t14 VALUES('one two three', 'one three four');
! 376: INSERT INTO t14 VALUES('a b c', 'd e a');
! 377: }
! 378: do_execsql_test 14.1 {
! 379: SELECT rowid FROM t14 WHERE t14 MATCH '"one two three"'
! 380: } {1}
! 381: do_execsql_test 14.2 {
! 382: SELECT rowid FROM t14 WHERE t14 MATCH '"one four"'
! 383: } {}
! 384: do_execsql_test 14.3 {
! 385: SELECT rowid FROM t14 WHERE t14 MATCH '"e a"'
! 386: } {2}
! 387: do_execsql_test 14.5 {
! 388: SELECT rowid FROM t14 WHERE t14 MATCH '"e b"'
! 389: } {}
! 390: do_catchsql_test 14.6 {
! 391: SELECT rowid FROM t14 WHERE rowid MATCH 'one'
! 392: } {1 {unable to use function MATCH in the requested context}}
! 393: do_catchsql_test 14.7 {
! 394: SELECT rowid FROM t14 WHERE docid MATCH 'one'
! 395: } {1 {unable to use function MATCH in the requested context}}
! 396:
! 397: do_execsql_test 15.0 {
! 398: CREATE VIRTUAL TABLE t15 USING fts4(a, b, c);
! 399: INSERT INTO t15 VALUES('abc def ghi', 'abc2 def2 ghi2', 'abc3 def3 ghi3');
! 400: INSERT INTO t15 VALUES('abc2 def2 ghi2', 'abc2 def2 ghi2', 'abc def3 ghi3');
! 401: }
! 402: do_execsql_test 15.1 {
! 403: SELECT rowid FROM t15 WHERE t15 MATCH '"abc* def2"'
! 404: } {1 2}
! 405:
! 406: # Test a corruption case.
! 407: #
! 408: do_execsql_test 16.1 {
! 409: CREATE VIRTUAL TABLE t16 USING fts4;
! 410: INSERT INTO t16 VALUES('theoretical work to examine the relationship');
! 411: INSERT INTO t16 VALUES('solution of our problems on the invisible');
! 412: DELETE FROM t16_content WHERE rowid = 2;
! 413: }
! 414: do_catchsql_test 16.2 {
! 415: SELECT * FROM t16 WHERE t16 MATCH 'invisible'
! 416: } {1 {database disk image is malformed}}
! 417:
! 418: # And another corruption test case.
! 419: #
! 420: do_execsql_test 17.1 {
! 421: CREATE VIRTUAL TABLE t17 USING fts4;
! 422: INSERT INTO t17(content) VALUES('one one one');
! 423: UPDATE t17_segdir SET root = X'00036F6E65FFFFFFFFFFFFFFFFFFFFFF02030300'
! 424: } {}
! 425: do_catchsql_test 17.2 {
! 426: SELECT * FROM t17 WHERE t17 MATCH 'one'
! 427: } {1 {database disk image is malformed}}
! 428:
! 429:
! 430:
! 431:
! 432: finish_test
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts3cov.test CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / test