1: # 2009 December 03
2: #
3: # May you do good and not evil.
4: # May you find forgiveness for yourself and forgive others.
5: # May you share freely, never taking more than you give.
6: #
7: #***********************************************************************
8: #
9: # The tests in this file are structural coverage tests for FTS3.
10: #
11:
12: set testdir [file dirname $argv0]
13: source $testdir/tester.tcl
14:
15: # If this build does not include FTS3, skip the tests in this file.
16: #
17: ifcapable !fts3 { finish_test ; return }
18: source $testdir/fts3_common.tcl
19: source $testdir/malloc_common.tcl
20:
21: set DO_MALLOC_TEST 0
22: set testprefix fts3cov
23:
24: #--------------------------------------------------------------------------
25: # When it first needs to read a block from the %_segments table, the FTS3
26: # module compiles an SQL statement for that purpose. The statement is
27: # stored and reused each subsequent time a block is read. This test case
28: # tests the effects of an OOM error occuring while compiling the statement.
29: #
30: # Similarly, when FTS3 first needs to scan through a set of segment leaves
31: # to find a set of documents that matches a term, it allocates a string
32: # containing the text of the required SQL, and compiles one or more
33: # statements to traverse the leaves. This test case tests that OOM errors
34: # that occur while allocating this string and statement are handled correctly
35: # also.
36: #
37: do_test fts3cov-1.1 {
38: execsql {
39: CREATE VIRTUAL TABLE t1 USING fts3(x);
40: INSERT INTO t1(t1) VALUES('nodesize=24');
41: BEGIN;
42: INSERT INTO t1 VALUES('Is the night chilly and dark?');
43: INSERT INTO t1 VALUES('The night is chilly, but not dark.');
44: INSERT INTO t1 VALUES('The thin gray cloud is spread on high,');
45: INSERT INTO t1 VALUES('It covers but not hides the sky.');
46: COMMIT;
47: SELECT count(*)>0 FROM t1_segments;
48: }
49: } {1}
50:
51: set DO_MALLOC_TEST 1
52: do_restart_select_test fts3cov-1.2 {
53: SELECT docid FROM t1 WHERE t1 MATCH 'chilly';
54: } {1 2}
55: set DO_MALLOC_TEST 0
56:
57: #--------------------------------------------------------------------------
58: # When querying the full-text index, if an expected internal node block is
59: # missing from the %_segments table, or if a NULL value is stored in the
60: # %_segments table instead of a binary blob, database corruption should be
61: # reported.
62: #
63: # Even with tiny 24 byte nodes, it takes a fair bit of data to produce a
64: # segment b-tree that uses the %_segments table to store internal nodes.
65: #
66: do_test fts3cov-2.1 {
67: execsql {
68: INSERT INTO t1(t1) VALUES('nodesize=24');
69: BEGIN;
70: INSERT INTO t1 VALUES('The moon is behind, and at the full;');
71: INSERT INTO t1 VALUES('And yet she looks both small and dull.');
72: INSERT INTO t1 VALUES('The night is chill, the cloud is gray:');
73: INSERT INTO t1 VALUES('''T is a month before the month of May,');
74: INSERT INTO t1 VALUES('And the Spring comes slowly up this way.');
75: INSERT INTO t1 VALUES('The lovely lady, Christabel,');
76: INSERT INTO t1 VALUES('Whom her father loves so well,');
77: INSERT INTO t1 VALUES('What makes her in the wood so late,');
78: INSERT INTO t1 VALUES('A furlong from the castle gate?');
79: INSERT INTO t1 VALUES('She had dreams all yesternight');
80: INSERT INTO t1 VALUES('Of her own betrothed knight;');
81: INSERT INTO t1 VALUES('And she in the midnight wood will pray');
82: INSERT INTO t1 VALUES('For the weal of her lover that''s far away.');
83: COMMIT;
84: }
85: execsql {
86: INSERT INTO t1(t1) VALUES('optimize');
87: SELECT substr(hex(root), 1, 2) FROM t1_segdir;
88: }
89: } {03}
90:
91: # Test the "missing entry" case:
92: do_test fts3cov-2.2 {
93: set root [db one {SELECT root FROM t1_segdir}]
94: read_fts3varint [string range $root 1 end] left_child
95: execsql { DELETE FROM t1_segments WHERE blockid = $left_child }
96: } {}
97: do_error_test fts3cov-2.3 {
98: SELECT * FROM t1 WHERE t1 MATCH 'c*'
99: } {SQL logic error or missing database}
100:
101: # Test the "replaced with NULL" case:
102: do_test fts3cov-2.4 {
103: execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }
104: } {}
105: do_error_test fts3cov-2.5 {
106: SELECT * FROM t1 WHERE t1 MATCH 'cloud'
107: } {SQL logic error or missing database}
108:
109: #--------------------------------------------------------------------------
110: # The following tests are to test the effects of OOM errors while storing
111: # terms in the pending-hash table. Specifically, while creating doclist
112: # blobs to store in the table. More specifically, to test OOM errors while
113: # appending column numbers to doclists. For example, if a doclist consists
114: # of:
115: #
116: # <docid> <column 0 offset-list> 0x01 <column N> <column N offset-list>
117: #
118: # The following tests check that malloc errors encountered while appending
119: # the "0x01 <column N>" data to the dynamically growable blob used to
120: # accumulate the doclist in memory are handled correctly.
121: #
122: do_test fts3cov-3.1 {
123: set cols [list]
124: set vals [list]
125: for {set i 0} {$i < 120} {incr i} {
126: lappend cols "col$i"
127: lappend vals "'word'"
128: }
129: execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"
130: } {}
131: set DO_MALLOC_TEST 1
132: do_write_test fts3cov-3.2 t2_content "
133: INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,])
134: "
135: do_write_test fts3cov-3.3 t2_content "
136: INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,])
137: "
138: do_write_test fts3cov-3.4 t2_content "
139: INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,])
140: "
141:
142: #-------------------------------------------------------------------------
143: # If too much data accumulates in the pending-terms hash table, it is
144: # flushed to the database automatically, even if the transaction has not
145: # finished. The following tests check the effects of encountering an OOM
146: # while doing this.
147: #
148: do_test fts3cov-4.1 {
149: execsql {
150: CREATE VIRTUAL TABLE t3 USING fts3(x);
151: INSERT INTO t3(t3) VALUES('nodesize=24');
152: INSERT INTO t3(t3) VALUES('maxpending=100');
153: }
154: } {}
155: set DO_MALLOC_TEST 1
156: do_write_test fts3cov-4.2 t3_content {
157: INSERT INTO t3(docid, x)
158: SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL
159: SELECT 3, 'And comforted fair Geraldine:' UNION ALL
160: SELECT 4, '''O well, bright dame, may you command' UNION ALL
161: SELECT 5, 'The service of Sir Leoline;' UNION ALL
162: SELECT 2, 'And gladly our stout chivalry' UNION ALL
163: SELECT 7, 'Will he send forth, and friends withal,' UNION ALL
164: SELECT 8, 'To guide and guard you safe and free' UNION ALL
165: SELECT 6, 'Home to your noble father''s hall.'''
166: }
167:
168: #-------------------------------------------------------------------------
169: # When building the internal tree structure for each segment b-tree, FTS3
170: # assumes that the content of each internal node will be less than
171: # $nodesize bytes, where $nodesize is the advisory node size. If this turns
172: # out to be untrue, then an extra buffer must be malloc'd for each term.
173: # This test case tests these paths and the effects of said mallocs failing
174: # by inserting insert a document with some fairly large terms into a
175: # full-text table with a very small node-size.
176: #
177: # Test this handling of large terms in three contexts:
178: #
179: # 1. When flushing the pending-terms table.
180: # 2. When optimizing the data structures using the INSERT syntax.
181: # 2. When optimizing the data structures using the deprecated SELECT syntax.
182: #
183: do_test fts3cov-5.1 {
184: execsql {
185: CREATE VIRTUAL TABLE t4 USING fts3(x);
186: INSERT INTO t4(t4) VALUES('nodesize=24');
187: }
188: } {}
189: set DO_MALLOC_TEST 1
190:
191: # Test when flushing pending-terms table.
192: do_write_test fts3cov-5.2 t4_content {
193: INSERT INTO t4
194: SELECT 'ItisanancientMarinerAndhestoppethoneofthreeAA' UNION ALL
195: SELECT 'ItisanancientMarinerAndhestoppethoneofthreeBB' UNION ALL
196: SELECT 'ItisanancientMarinerAndhestoppethoneofthreeCC' UNION ALL
197: SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstAA' UNION ALL
198: SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstBB' UNION ALL
199: SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstCC'
200: }
201:
202: # Test when optimizing via INSERT.
203: do_test fts3cov-5.3 { execsql { INSERT INTO t4 VALUES('extra!') } } {}
204: do_write_test fts3cov-5.2 t4_segments { INSERT INTO t4(t4) VALUES('optimize') }
205:
206: # Test when optimizing via SELECT.
207: do_test fts3cov-5.5 { execsql { INSERT INTO t4 VALUES('more extra!') } } {}
208: do_write_test fts3cov-5.6 t4_segments {
209: SELECT * FROM (SELECT optimize(t4) FROM t4 LIMIT 1)
210: EXCEPT SELECT 'Index optimized'
211: }
212:
213: #-------------------------------------------------------------------------
214: # When merging all segments at a given level to create a single segment
215: # at level+1, FTS3 runs a query of the form:
216: #
217: # SELECT count(*) FROM %_segdir WHERE level = ?
218: #
219: # The query is compiled the first time this operation is required and
220: # reused thereafter. This test aims to test the effects of an OOM while
221: # preparing and executing this query for the first time.
222: #
223: # Then, keep inserting rows into the table so that the effects of an OOM
224: # while re-executing the same query can also be tested.
225: #
226: do_test fts3cov-6.1 {
227: execsql { CREATE VIRTUAL TABLE t5 USING fts3(x) }
228: for {set i 0} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
229: execsql { SELECT count(*) FROM t5_segdir }
230: } {16}
231:
232: # First time.
233: db close
234: sqlite3 db test.db
235: do_write_test fts3cov-6.2 t5_content {
236: INSERT INTO t5 VALUES('segment number 16!');
237: }
238:
239: # Second time.
240: do_test fts3cov-6.3 {
241: for {set i 1} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
242: execsql { SELECT count(*) FROM t5_segdir }
243: } {17}
244: do_write_test fts3cov-6.4 t5_content {
245: INSERT INTO t5 VALUES('segment number 16!');
246: }
247:
248: #-------------------------------------------------------------------------
249: # Update the docid of a row. Test this in two scenarios:
250: #
251: # 1. When the row being updated is the only row in the table.
252: # 2. When it is not.
253: #
254: # The two cases above take different paths because in case 1 all data
255: # structures can simply be emptied before inserting the new row record.
256: # In case 2, the data structures actually have to be updated.
257: #
258: do_test fts3cov-7.1 {
259: execsql {
260: CREATE VIRTUAL TABLE t7 USING fts3(a, b, c);
261: INSERT INTO t7 VALUES('A', 'B', 'C');
262: UPDATE t7 SET docid = 5;
263: SELECT docid, * FROM t7;
264: }
265: } {5 A B C}
266: do_test fts3cov-7.2 {
267: execsql {
268: INSERT INTO t7 VALUES('D', 'E', 'F');
269: UPDATE t7 SET docid = 1 WHERE docid = 6;
270: SELECT docid, * FROM t7;
271: }
272: } {1 D E F 5 A B C}
273:
274: #-------------------------------------------------------------------------
275: # If a set of documents are modified within a transaction, the
276: # pending-terms table must be flushed each time a document with a docid
277: # less than or equal to the previous docid is modified.
278: #
279: # This test checks the effects of an OOM error occuring when the
280: # pending-terms table is flushed for this reason as part of a DELETE
281: # statement.
282: #
283: do_malloc_test fts3cov-8 -sqlprep {
284: BEGIN;
285: CREATE VIRTUAL TABLE t8 USING fts3;
286: INSERT INTO t8 VALUES('the output of each batch run');
287: INSERT INTO t8 VALUES('(possibly a day''s work)');
288: INSERT INTO t8 VALUES('was written to two separate disks');
289: COMMIT;
290: } -sqlbody {
291: BEGIN;
292: DELETE FROM t8 WHERE rowid = 3;
293: DELETE FROM t8 WHERE rowid = 2;
294: DELETE FROM t8 WHERE rowid = 1;
295: COMMIT;
296: }
297:
298: #-------------------------------------------------------------------------
299: # Test some branches in the code that handles "special" inserts like:
300: #
301: # INSERT INTO t1(t1) VALUES('optimize');
302: #
303: # Also test that an optimize (INSERT method) works on an empty table.
304: #
305: set DO_MALLOC_TEST 0
306: do_test fts3cov-9.1 {
307: execsql { CREATE VIRTUAL TABLE xx USING fts3 }
308: } {}
309: do_error_test fts3cov-9.2 {
310: INSERT INTO xx(xx) VALUES('optimise'); -- British spelling
311: } {SQL logic error or missing database}
312: do_error_test fts3cov-9.3 {
313: INSERT INTO xx(xx) VALUES('short');
314: } {SQL logic error or missing database}
315: do_error_test fts3cov-9.4 {
316: INSERT INTO xx(xx) VALUES('waytoolongtobecorrect');
317: } {SQL logic error or missing database}
318: do_test fts3cov-9.5 {
319: execsql { INSERT INTO xx(xx) VALUES('optimize') }
320: } {}
321:
322: #-------------------------------------------------------------------------
323: # Test that a table can be optimized in the middle of a transaction when
324: # the pending-terms table is non-empty. This case involves some extra
325: # branches because data must be read not only from the database, but
326: # also from the pending-terms table.
327: #
328: do_malloc_test fts3cov-10 -sqlprep {
329: CREATE VIRTUAL TABLE t10 USING fts3;
330: INSERT INTO t10 VALUES('Optimising images for the web is a tricky business');
331: BEGIN;
332: INSERT INTO t10 VALUES('You have to get the right balance between');
333: } -sqlbody {
334: INSERT INTO t10(t10) VALUES('optimize');
335: }
336:
337: #-------------------------------------------------------------------------
338: # Test a full-text query for a term that was once in the index, but is
339: # no longer.
340: #
341: do_test fts3cov-11.1 {
342: execsql {
343: CREATE VIRTUAL TABLE xx USING fts3;
344: INSERT INTO xx VALUES('one two three');
345: INSERT INTO xx VALUES('four five six');
346: DELETE FROM xx WHERE docid = 1;
347: }
348: execsql { SELECT * FROM xx WHERE xx MATCH 'two' }
349: } {}
350:
351:
352: do_malloc_test fts3cov-12 -sqlprep {
353: CREATE VIRTUAL TABLE t12 USING fts3;
354: INSERT INTO t12 VALUES('is one of the two togther');
355: BEGIN;
356: INSERT INTO t12 VALUES('one which was appropriate at the time');
357: } -sqlbody {
358: SELECT * FROM t12 WHERE t12 MATCH 'one'
359: }
360:
361: do_malloc_test fts3cov-13 -sqlprep {
362: PRAGMA encoding = 'UTF-16';
363: CREATE VIRTUAL TABLE t13 USING fts3;
364: INSERT INTO t13 VALUES('two scalar functions');
365: INSERT INTO t13 VALUES('scalar two functions');
366: INSERT INTO t13 VALUES('functions scalar two');
367: } -sqlbody {
368: SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two';
369: SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two';
370: SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two';
371: }
372:
373: do_execsql_test 14.0 {
374: CREATE VIRTUAL TABLE t14 USING fts4(a, b);
375: INSERT INTO t14 VALUES('one two three', 'one three four');
376: INSERT INTO t14 VALUES('a b c', 'd e a');
377: }
378: do_execsql_test 14.1 {
379: SELECT rowid FROM t14 WHERE t14 MATCH '"one two three"'
380: } {1}
381: do_execsql_test 14.2 {
382: SELECT rowid FROM t14 WHERE t14 MATCH '"one four"'
383: } {}
384: do_execsql_test 14.3 {
385: SELECT rowid FROM t14 WHERE t14 MATCH '"e a"'
386: } {2}
387: do_execsql_test 14.5 {
388: SELECT rowid FROM t14 WHERE t14 MATCH '"e b"'
389: } {}
390: do_catchsql_test 14.6 {
391: SELECT rowid FROM t14 WHERE rowid MATCH 'one'
392: } {1 {unable to use function MATCH in the requested context}}
393: do_catchsql_test 14.7 {
394: SELECT rowid FROM t14 WHERE docid MATCH 'one'
395: } {1 {unable to use function MATCH in the requested context}}
396:
397: do_execsql_test 15.0 {
398: CREATE VIRTUAL TABLE t15 USING fts4(a, b, c);
399: INSERT INTO t15 VALUES('abc def ghi', 'abc2 def2 ghi2', 'abc3 def3 ghi3');
400: INSERT INTO t15 VALUES('abc2 def2 ghi2', 'abc2 def2 ghi2', 'abc def3 ghi3');
401: }
402: do_execsql_test 15.1 {
403: SELECT rowid FROM t15 WHERE t15 MATCH '"abc* def2"'
404: } {1 2}
405:
406: # Test a corruption case.
407: #
408: do_execsql_test 16.1 {
409: CREATE VIRTUAL TABLE t16 USING fts4;
410: INSERT INTO t16 VALUES('theoretical work to examine the relationship');
411: INSERT INTO t16 VALUES('solution of our problems on the invisible');
412: DELETE FROM t16_content WHERE rowid = 2;
413: }
414: do_catchsql_test 16.2 {
415: SELECT * FROM t16 WHERE t16 MATCH 'invisible'
416: } {1 {database disk image is malformed}}
417:
418: # And another corruption test case.
419: #
420: do_execsql_test 17.1 {
421: CREATE VIRTUAL TABLE t17 USING fts4;
422: INSERT INTO t17(content) VALUES('one one one');
423: UPDATE t17_segdir SET root = X'00036F6E65FFFFFFFFFFFFFFFFFFFFFF02030300'
424: } {}
425: do_catchsql_test 17.2 {
426: SELECT * FROM t17 WHERE t17 MATCH 'one'
427: } {1 {database disk image is malformed}}
428:
429:
430:
431:
432: finish_test
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts3cov.test CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / test