Annotation of embedaddon/sqlite3/test/fts3rnd.test, revision 1.1.1.1
1.1 misho 1: # 2009 December 03
2: #
3: # May you do good and not evil.
4: # May you find forgiveness for yourself and forgive others.
5: # May you share freely, never taking more than you give.
6: #
7: #***********************************************************************
8: #
9: # Brute force (random data) tests for FTS3.
10: #
11:
12: #-------------------------------------------------------------------------
13: #
14: # The FTS3 tests implemented in this file focus on testing that FTS3
15: # returns the correct set of documents for various types of full-text
16: # query. This is done using pseudo-randomly generated data and queries.
17: # The expected result of each query is calculated using Tcl code.
18: #
19: # 1. The database is initialized to contain a single table with three
20: # columns. 100 rows are inserted into the table. Each of the three
21: # values in each row is a document consisting of between 0 and 100
22: # terms. Terms are selected from a vocabulary of $G(nVocab) terms.
23: #
24: # 2. The following is performed 100 times:
25: #
26: # a. A row is inserted into the database. The row contents are
27: # generated as in step 1. The docid is a pseudo-randomly selected
28: # value between 0 and 1000000.
29: #
30: # b. A psuedo-randomly selected row is updated. One of its columns is
31: # set to contain a new document generated in the same way as the
32: # documents in step 1.
33: #
34: # c. A psuedo-randomly selected row is deleted.
35: #
36: # d. For each of several types of fts3 queries, 10 SELECT queries
37: # of the form:
38: #
39: # SELECT docid FROM <tbl> WHERE <tbl> MATCH '<query>'
40: #
41: # are evaluated. The results are compared to those calculated by
42: # Tcl code in this file. The patterns used for the different query
43: # types are:
44: #
45: # 1. query = <term>
46: # 2. query = <prefix>
47: # 3. query = "<term> <term>"
48: # 4. query = "<term> <term> <term>"
49: # 5. query = "<prefix> <prefix> <prefix>"
50: # 6. query = <term> NEAR <term>
51: # 7. query = <term> NEAR/11 <term> NEAR/11 <term>
52: # 8. query = <term> OR <term>
53: # 9. query = <term> NOT <term>
54: # 10. query = <term> AND <term>
55: # 11. query = <term> NEAR <term> OR <term> NEAR <term>
56: # 12. query = <term> NEAR <term> NOT <term> NEAR <term>
57: # 13. query = <term> NEAR <term> AND <term> NEAR <term>
58: #
59: # where <term> is a term psuedo-randomly selected from the vocabulary
60: # and prefix is the first 2 characters of such a term followed by
61: # a "*" character.
62: #
63: # Every second iteration, steps (a) through (d) above are performed
64: # within a single transaction. This forces the queries in (d) to
65: # read data from both the database and the in-memory hash table
66: # that caches the full-text index entries created by steps (a), (b)
67: # and (c) until the transaction is committed.
68: #
69: # The procedure above is run 5 times, using advisory fts3 node sizes of 50,
70: # 500, 1000 and 2000 bytes.
71: #
72: # After the test using an advisory node-size of 50, an OOM test is run using
73: # the database. This test is similar to step (d) above, except that it tests
74: # the effects of transient and persistent OOM conditions encountered while
75: # executing each query.
76: #
77:
78: set testdir [file dirname $argv0]
79: source $testdir/tester.tcl
80:
81: # If this build does not include FTS3, skip the tests in this file.
82: #
83: ifcapable !fts3 { finish_test ; return }
84: source $testdir/fts3_common.tcl
85: source $testdir/malloc_common.tcl
86:
87: set G(nVocab) 100
88:
89: set nVocab 100
90: set lVocab [list]
91:
92: expr srand(0)
93:
94: # Generate a vocabulary of nVocab words. Each word is 3 characters long.
95: #
96: set lChar {a b c d e f g h i j k l m n o p q r s t u v w x y z}
97: for {set i 0} {$i < $nVocab} {incr i} {
98: set len [expr int(rand()*3)+2]
99: set word [lindex $lChar [expr int(rand()*26)]]
100: append word [lindex $lChar [expr int(rand()*26)]]
101: if {$len>2} { append word [lindex $lChar [expr int(rand()*26)]] }
102: if {$len>3} { append word [lindex $lChar [expr int(rand()*26)]] }
103: lappend lVocab $word
104: }
105:
106: proc random_term {} {
107: lindex $::lVocab [expr {int(rand()*$::nVocab)}]
108: }
109:
110: # Return a document consisting of $nWord arbitrarily selected terms
111: # from the $::lVocab list.
112: #
113: proc generate_doc {nWord} {
114: set doc [list]
115: for {set i 0} {$i < $nWord} {incr i} {
116: lappend doc [random_term]
117: }
118: return $doc
119: }
120:
121:
122:
123: # Primitives to update the table.
124: #
125: unset -nocomplain t1
126: proc insert_row {rowid} {
127: set a [generate_doc [expr int((rand()*100))]]
128: set b [generate_doc [expr int((rand()*100))]]
129: set c [generate_doc [expr int((rand()*100))]]
130: execsql { INSERT INTO t1(docid, a, b, c) VALUES($rowid, $a, $b, $c) }
131: set ::t1($rowid) [list $a $b $c]
132: }
133: proc delete_row {rowid} {
134: execsql { DELETE FROM t1 WHERE rowid = $rowid }
135: catch {unset ::t1($rowid)}
136: }
137: proc update_row {rowid} {
138: set cols {a b c}
139: set iCol [expr int(rand()*3)]
140: set doc [generate_doc [expr int((rand()*100))]]
141: lset ::t1($rowid) $iCol $doc
142: execsql "UPDATE t1 SET [lindex $cols $iCol] = \$doc WHERE rowid = \$rowid"
143: }
144:
145: proc simple_phrase {zPrefix} {
146: set ret [list]
147:
148: set reg [string map {* {[^ ]*}} $zPrefix]
149: set reg " $reg "
150:
151: foreach key [lsort -integer [array names ::t1]] {
152: set value $::t1($key)
153: set cnt [list]
154: foreach col $value {
155: if {[regexp $reg " $col "]} { lappend ret $key ; break }
156: }
157: }
158:
159: #lsort -uniq -integer $ret
160: set ret
161: }
162:
163: # This [proc] is used to test the FTS3 matchinfo() function.
164: #
165: proc simple_token_matchinfo {zToken bDesc} {
166:
167: set nDoc(0) 0
168: set nDoc(1) 0
169: set nDoc(2) 0
170: set nHit(0) 0
171: set nHit(1) 0
172: set nHit(2) 0
173:
174: set dir -inc
175: if {$bDesc} { set dir -dec }
176:
177: foreach key [array names ::t1] {
178: set value $::t1($key)
179: set a($key) [list]
180: foreach i {0 1 2} col $value {
181: set hit [llength [lsearch -all $col $zToken]]
182: lappend a($key) $hit
183: incr nHit($i) $hit
184: if {$hit>0} { incr nDoc($i) }
185: }
186: }
187:
188: set ret [list]
189: foreach docid [lsort -integer $dir [array names a]] {
190: if { [lindex [lsort -integer $a($docid)] end] } {
191: set matchinfo [list 1 3]
192: foreach i {0 1 2} hit $a($docid) {
193: lappend matchinfo $hit $nHit($i) $nDoc($i)
194: }
195: lappend ret $docid $matchinfo
196: }
197: }
198:
199: set ret
200: }
201:
202: proc simple_near {termlist nNear} {
203: set ret [list]
204:
205: foreach {key value} [array get ::t1] {
206: foreach v $value {
207:
208: set l [lsearch -exact -all $v [lindex $termlist 0]]
209: foreach T [lrange $termlist 1 end] {
210: set l2 [list]
211: foreach i $l {
212: set iStart [expr $i - $nNear - 1]
213: set iEnd [expr $i + $nNear + 1]
214: if {$iStart < 0} {set iStart 0}
215: foreach i2 [lsearch -exact -all [lrange $v $iStart $iEnd] $T] {
216: incr i2 $iStart
217: if {$i2 != $i} { lappend l2 $i2 }
218: }
219: }
220: set l [lsort -uniq -integer $l2]
221: }
222:
223: if {[llength $l]} {
224: #puts "MATCH($key): $v"
225: lappend ret $key
226: }
227: }
228: }
229:
230: lsort -unique -integer $ret
231: }
232:
233: # The following three procs:
234: #
235: # setup_not A B
236: # setup_or A B
237: # setup_and A B
238: #
239: # each take two arguments. Both arguments must be lists of integer values
240: # sorted by value. The return value is the list produced by evaluating
241: # the equivalent of "A op B", where op is the FTS3 operator NOT, OR or
242: # AND.
243: #
244: proc setop_not {A B} {
245: foreach b $B { set n($b) {} }
246: set ret [list]
247: foreach a $A { if {![info exists n($a)]} {lappend ret $a} }
248: return $ret
249: }
250: proc setop_or {A B} {
251: lsort -integer -uniq [concat $A $B]
252: }
253: proc setop_and {A B} {
254: foreach b $B { set n($b) {} }
255: set ret [list]
256: foreach a $A { if {[info exists n($a)]} {lappend ret $a} }
257: return $ret
258: }
259:
260: proc mit {blob} {
261: set scan(littleEndian) i*
262: set scan(bigEndian) I*
263: binary scan $blob $scan($::tcl_platform(byteOrder)) r
264: return $r
265: }
266: db func mit mit
267: set sqlite_fts3_enable_parentheses 1
268:
269: proc do_orderbydocid_test {tn sql res} {
270: uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res]
271: uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \
272: [lsort -int -dec $res]
273: ]
274: }
275:
276: set NUM_TRIALS 100
277:
278: foreach {nodesize order} {
279: 50 DESC
280: 50 ASC
281: 500 ASC
282: 1000 DESC
283: 2000 ASC
284: } {
285: catch { array unset ::t1 }
286: set testname "$nodesize/$order"
287:
288: # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows.
289: #
290: db transaction {
291: catchsql { DROP TABLE t1 }
292: execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)"
293: execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')"
294: for {set i 0} {$i < 100} {incr i} { insert_row $i }
295: }
296:
297: for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} {
298: catchsql COMMIT
299:
300: set DO_MALLOC_TEST 0
301: set nRep 10
302: if {$iTest==100 && $nodesize==50} {
303: set DO_MALLOC_TEST 1
304: set nRep 2
305: }
306:
307: set ::testprefix fts3rnd-1.$testname.$iTest
308:
309: # Delete one row, update one row and insert one row.
310: #
311: set rows [array names ::t1]
312: set nRow [llength $rows]
313: set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]]
314: set iDelete $iUpdate
315: while {$iDelete == $iUpdate} {
316: set iDelete [lindex $rows [expr {int(rand()*$nRow)}]]
317: }
318: set iInsert $iUpdate
319: while {[info exists ::t1($iInsert)]} {
320: set iInsert [expr {int(rand()*1000000)}]
321: }
322: execsql BEGIN
323: insert_row $iInsert
324: update_row $iUpdate
325: delete_row $iDelete
326: if {0==($iTest%2)} { execsql COMMIT }
327:
328: if {0==($iTest%2)} {
329: #do_test 0 { fts3_integrity_check t1 } ok
330: }
331:
332: # Pick 10 terms from the vocabulary. Check that the results of querying
333: # the database for the set of documents containing each of these terms
334: # is the same as the result obtained by scanning the contents of the Tcl
335: # array for each term.
336: #
337: for {set i 0} {$i < 10} {incr i} {
338: set term [random_term]
339: do_select_test 1.$i.asc {
340: SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
341: ORDER BY docid ASC
342: } [simple_token_matchinfo $term 0]
343: do_select_test 1.$i.desc {
344: SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
345: ORDER BY docid DESC
346: } [simple_token_matchinfo $term 1]
347: }
348:
349: # This time, use the first two characters of each term as a term prefix
350: # to query for. Test that querying the Tcl array produces the same results
351: # as querying the FTS3 table for the prefix.
352: #
353: for {set i 0} {$i < $nRep} {incr i} {
354: set prefix [string range [random_term] 0 end-1]
355: set match "${prefix}*"
356: do_orderbydocid_test 2.$i {
357: SELECT docid FROM t1 WHERE t1 MATCH $match
358: } [simple_phrase $match]
359: }
360:
361: # Similar to the above, except for phrase queries.
362: #
363: for {set i 0} {$i < $nRep} {incr i} {
364: set term [list [random_term] [random_term]]
365: set match "\"$term\""
366: do_orderbydocid_test 3.$i {
367: SELECT docid FROM t1 WHERE t1 MATCH $match
368: } [simple_phrase $term]
369: }
370:
371: # Three word phrases.
372: #
373: for {set i 0} {$i < $nRep} {incr i} {
374: set term [list [random_term] [random_term] [random_term]]
375: set match "\"$term\""
376: do_orderbydocid_test 4.$i {
377: SELECT docid FROM t1 WHERE t1 MATCH $match
378: } [simple_phrase $term]
379: }
380:
381: # Three word phrases made up of term-prefixes.
382: #
383: for {set i 0} {$i < $nRep} {incr i} {
384: set query "[string range [random_term] 0 end-1]* "
385: append query "[string range [random_term] 0 end-1]* "
386: append query "[string range [random_term] 0 end-1]*"
387:
388: set match "\"$query\""
389: do_orderbydocid_test 5.$i {
390: SELECT docid FROM t1 WHERE t1 MATCH $match
391: } [simple_phrase $query]
392: }
393:
394: # A NEAR query with terms as the arguments:
395: #
396: # ... MATCH '$term1 NEAR $term2' ...
397: #
398: for {set i 0} {$i < $nRep} {incr i} {
399: set terms [list [random_term] [random_term]]
400: set match [join $terms " NEAR "]
401: do_orderbydocid_test 6.$i {
402: SELECT docid FROM t1 WHERE t1 MATCH $match
403: } [simple_near $terms 10]
404: }
405:
406: # A 3-way NEAR query with terms as the arguments.
407: #
408: for {set i 0} {$i < $nRep} {incr i} {
409: set terms [list [random_term] [random_term] [random_term]]
410: set nNear 11
411: set match [join $terms " NEAR/$nNear "]
412: do_orderbydocid_test 7.$i {
413: SELECT docid FROM t1 WHERE t1 MATCH $match
414: } [simple_near $terms $nNear]
415: }
416:
417: # Set operations on simple term queries.
418: #
419: foreach {tn op proc} {
420: 8 OR setop_or
421: 9 NOT setop_not
422: 10 AND setop_and
423: } {
424: for {set i 0} {$i < $nRep} {incr i} {
425: set term1 [random_term]
426: set term2 [random_term]
427: set match "$term1 $op $term2"
428: do_orderbydocid_test $tn.$i {
429: SELECT docid FROM t1 WHERE t1 MATCH $match
430: } [$proc [simple_phrase $term1] [simple_phrase $term2]]
431: }
432: }
433:
434: # Set operations on NEAR queries.
435: #
436: foreach {tn op proc} {
437: 11 OR setop_or
438: 12 NOT setop_not
439: 13 AND setop_and
440: } {
441: for {set i 0} {$i < $nRep} {incr i} {
442: set term1 [random_term]
443: set term2 [random_term]
444: set term3 [random_term]
445: set term4 [random_term]
446: set match "$term1 NEAR $term2 $op $term3 NEAR $term4"
447: do_orderbydocid_test $tn.$i {
448: SELECT docid FROM t1 WHERE t1 MATCH $match
449: } [$proc \
450: [simple_near [list $term1 $term2] 10] \
451: [simple_near [list $term3 $term4] 10]
452: ]
453: }
454: }
455:
456: catchsql COMMIT
457: }
458: }
459:
460: finish_test
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to fts3rnd.test CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / test