1: # 2010 April 13
2: #
3: # The author disclaims copyright to this source code. In place of
4: # a legal notice, here is a blessing:
5: #
6: # May you do good and not evil.
7: # May you find forgiveness for yourself and forgive others.
8: # May you share freely, never taking more than you give.
9: #
10: #***********************************************************************
11: # This file implements regression tests for SQLite library. The
12: # focus of this file is testing the operation of the library in
13: # "PRAGMA journal_mode=WAL" mode with multiple threads.
14: #
15:
16: set testdir [file dirname $argv0]
17:
18: source $testdir/tester.tcl
19: source $testdir/lock_common.tcl
20: if {[run_thread_tests]==0} { finish_test ; return }
21: ifcapable !wal { finish_test ; return }
22:
23: set sqlite_walsummary_mmap_incr 64
24:
25: # How long, in seconds, to run each test for. If a test is set to run for
26: # 0 seconds, it is omitted entirely.
27: #
28: unset -nocomplain seconds
29: set seconds(walthread-1) 20
30: set seconds(walthread-2) 20
31: set seconds(walthread-3) 20
32: set seconds(walthread-4) 20
33: set seconds(walthread-5) 1
34:
35: # The parameter is the name of a variable in the callers context. The
36: # variable may or may not exist when this command is invoked.
37: #
38: # If the variable does exist, its value is returned. Otherwise, this
39: # command uses [vwait] to wait until it is set, then returns the value.
40: # In other words, this is a version of the [set VARNAME] command that
41: # blocks until a variable exists.
42: #
43: proc wait_for_var {varname} {
44: if {0==[uplevel [list info exists $varname]]} {
45: uplevel [list vwait $varname]
46: }
47: uplevel [list set $varname]
48: }
49:
50: # The argument is the name of a list variable in the callers context. The
51: # first element of the list is removed and returned. For example:
52: #
53: # set L {a b c}
54: # set x [lshift L]
55: # assert { $x == "a" && $L == "b c" }
56: #
57: proc lshift {lvar} {
58: upvar $lvar L
59: set ret [lindex $L 0]
60: set L [lrange $L 1 end]
61: return $ret
62: }
63:
64:
65: #-------------------------------------------------------------------------
66: # do_thread_test TESTNAME OPTIONS...
67: #
68: # where OPTIONS are:
69: #
70: # -seconds SECONDS How many seconds to run the test for
71: # -init SCRIPT Script to run before test.
72: # -thread NAME COUNT SCRIPT Scripts to run in threads (or processes).
73: # -processes BOOLEAN True to use processes instead of threads.
74: # -check SCRIPT Script to run after test.
75: #
76: proc do_thread_test {args} {
77:
78: set A $args
79:
80: set P(testname) [lshift A]
81: set P(seconds) 5
82: set P(init) ""
83: set P(threads) [list]
84: set P(processes) 0
85: set P(check) {
86: set ic [db eval "PRAGMA integrity_check"]
87: if {$ic != "ok"} { error $ic }
88: }
89:
90: unset -nocomplain ::done
91:
92: while {[llength $A]>0} {
93: set a [lshift A]
94: switch -glob -- $a {
95: -seconds {
96: set P(seconds) [lshift A]
97: }
98:
99: -init {
100: set P(init) [lshift A]
101: }
102:
103: -processes {
104: set P(processes) [lshift A]
105: }
106:
107: -check {
108: set P(check) [lshift A]
109: }
110:
111: -thread {
112: set name [lshift A]
113: set count [lshift A]
114: set prg [lshift A]
115: lappend P(threads) [list $name $count $prg]
116: }
117:
118: default {
119: error "Unknown option: $a"
120: }
121: }
122: }
123:
124: if {$P(seconds) == 0} {
125: puts "Skipping $P(testname)"
126: return
127: }
128:
129: puts "Running $P(testname) for $P(seconds) seconds..."
130:
131: catch { db close }
132: forcedelete test.db test.db-journal test.db-wal
133:
134: sqlite3 db test.db
135: eval $P(init)
136: catch { db close }
137:
138: foreach T $P(threads) {
139: set name [lindex $T 0]
140: set count [lindex $T 1]
141: set prg [lindex $T 2]
142:
143: for {set i 1} {$i <= $count} {incr i} {
144: set vars "
145: set E(pid) $i
146: set E(nthread) $count
147: set E(seconds) $P(seconds)
148: "
149: set program [string map [list %TEST% $prg %VARS% $vars] {
150:
151: %VARS%
152:
153: proc usleep {ms} {
154: set ::usleep 0
155: after $ms {set ::usleep 1}
156: vwait ::usleep
157: }
158:
159: proc integrity_check {{db db}} {
160: set ic [$db eval {PRAGMA integrity_check}]
161: if {$ic != "ok"} {error $ic}
162: }
163:
164: proc busyhandler {n} { usleep 10 ; return 0 }
165:
166: sqlite3 db test.db
167: db busy busyhandler
168: db eval { SELECT randomblob($E(pid)*5) }
169:
170: set ::finished 0
171: after [expr $E(seconds) * 1000] {set ::finished 1}
172: proc tt_continue {} { update ; expr ($::finished==0) }
173:
174: set rc [catch { %TEST% } msg]
175:
176: catch { db close }
177: list $rc $msg
178: }]
179:
180: if {$P(processes)==0} {
181: sqlthread spawn ::done($name,$i) $program
182: } else {
183: testfixture_nb ::done($name,$i) $program
184: }
185: }
186: }
187:
188: set report " Results:"
189: foreach T $P(threads) {
190: set name [lindex $T 0]
191: set count [lindex $T 1]
192: set prg [lindex $T 2]
193:
194: set reslist [list]
195: for {set i 1} {$i <= $count} {incr i} {
196: set res [wait_for_var ::done($name,$i)]
197: lappend reslist [lindex $res 1]
198: do_test $P(testname).$name.$i [list lindex $res 0] 0
199: }
200:
201: append report " $name $reslist"
202: }
203: puts $report
204:
205: sqlite3 db test.db
206: set res ""
207: if {[catch $P(check) msg]} { set res $msg }
208: do_test $P(testname).check [list set {} $res] ""
209: }
210:
211: # A wrapper around [do_thread_test] which runs the specified test twice.
212: # Once using processes, once using threads. This command takes the same
213: # arguments as [do_thread_test], except specifying the -processes switch
214: # is illegal.
215: #
216: proc do_thread_test2 {args} {
217: set name [lindex $args 0]
218: if {[lsearch $args -processes]>=0} { error "bad option: -processes"}
219: uplevel [lreplace $args 0 0 do_thread_test "$name-threads" -processes 0]
220: uplevel [lreplace $args 0 0 do_thread_test "$name-processes" -processes 1]
221: }
222:
223: #--------------------------------------------------------------------------
224: # Start 10 threads. Each thread performs both read and write
225: # transactions. Each read transaction consists of:
226: #
227: # 1) Reading the md5sum of all but the last table row,
228: # 2) Running integrity check.
229: # 3) Reading the value stored in the last table row,
230: # 4) Check that the values read in steps 1 and 3 are the same, and that
231: # the md5sum of all but the last table row has not changed.
232: #
233: # Each write transaction consists of:
234: #
235: # 1) Modifying the contents of t1 (inserting, updating, deleting rows).
236: # 2) Appending a new row to the table containing the md5sum() of all
237: # rows in the table.
238: #
239: # Each of the N threads runs N read transactions followed by a single write
240: # transaction in a loop as fast as possible.
241: #
242: # There is also a single checkpointer thread. It runs the following loop:
243: #
244: # 1) Execute "PRAGMA wal_checkpoint"
245: # 2) Sleep for 500 ms.
246: #
247: do_thread_test2 walthread-1 -seconds $seconds(walthread-1) -init {
248: execsql {
249: PRAGMA journal_mode = WAL;
250: CREATE TABLE t1(x PRIMARY KEY);
251: PRAGMA lock_status;
252: INSERT INTO t1 VALUES(randomblob(100));
253: INSERT INTO t1 VALUES(randomblob(100));
254: INSERT INTO t1 SELECT md5sum(x) FROM t1;
255: }
256: } -thread main 10 {
257:
258: proc read_transaction {} {
259: set results [db eval {
260: BEGIN;
261: PRAGMA integrity_check;
262: SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1);
263: SELECT x FROM t1 WHERE rowid = (SELECT max(rowid) FROM t1);
264: SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1);
265: COMMIT;
266: }]
267:
268: if {[llength $results]!=4
269: || [lindex $results 0] != "ok"
270: || [lindex $results 1] != [lindex $results 2]
271: || [lindex $results 2] != [lindex $results 3]
272: } {
273: error "Failed read transaction: $results"
274: }
275: }
276:
277: proc write_transaction {} {
278: db eval {
279: BEGIN;
280: INSERT INTO t1 VALUES(randomblob(100));
281: INSERT INTO t1 VALUES(randomblob(100));
282: INSERT INTO t1 SELECT md5sum(x) FROM t1;
283: COMMIT;
284: }
285: }
286:
287: # Turn off auto-checkpoint. Otherwise, an auto-checkpoint run by a
288: # writer may cause the dedicated checkpoint thread to return an
289: # SQLITE_BUSY error.
290: #
291: db eval { PRAGMA wal_autocheckpoint = 0 }
292:
293: set nRun 0
294: while {[tt_continue]} {
295: read_transaction
296: write_transaction
297: incr nRun
298: }
299: set nRun
300:
301: } -thread ckpt 1 {
302: set nRun 0
303: while {[tt_continue]} {
304: db eval "PRAGMA wal_checkpoint"
305: usleep 500
306: incr nRun
307: }
308: set nRun
309: }
310:
311: #--------------------------------------------------------------------------
312: # This test has clients run the following procedure as fast as possible
313: # in a loop:
314: #
315: # 1. Open a database handle.
316: # 2. Execute a read-only transaction on the db.
317: # 3. Do "PRAGMA journal_mode = XXX", where XXX is one of WAL or DELETE.
318: # Ignore any SQLITE_BUSY error.
319: # 4. Execute a write transaction to insert a row into the db.
320: # 5. Run "PRAGMA integrity_check"
321: #
322: # At present, there are 4 clients in total. 2 do "journal_mode = WAL", and
323: # two do "journal_mode = DELETE".
324: #
325: # Each client returns a string of the form "W w, R r", where W is the
326: # number of write-transactions performed using a WAL journal, and D is
327: # the number of write-transactions performed using a rollback journal.
328: # For example, "192 w, 185 r".
329: #
330: do_thread_test2 walthread-2 -seconds $seconds(walthread-2) -init {
331: execsql { CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE) }
332: } -thread RB 2 {
333:
334: db close
335: set nRun 0
336: set nDel 0
337: while {[tt_continue]} {
338: sqlite3 db test.db
339: db busy busyhandler
340: db eval { SELECT * FROM sqlite_master }
341: catch { db eval { PRAGMA journal_mode = DELETE } }
342: db eval {
343: BEGIN;
344: INSERT INTO t1 VALUES(NULL, randomblob(100+$E(pid)));
345: }
346: incr nRun 1
347: incr nDel [file exists test.db-journal]
348: if {[file exists test.db-journal] + [file exists test.db-wal] != 1} {
349: error "File-system looks bad..."
350: }
351: db eval COMMIT
352:
353: integrity_check
354: db close
355: }
356: list $nRun $nDel
357: set {} "[expr $nRun-$nDel] w, $nDel r"
358:
359: } -thread WAL 2 {
360: db close
361: set nRun 0
362: set nDel 0
363: while {[tt_continue]} {
364: sqlite3 db test.db
365: db busy busyhandler
366: db eval { SELECT * FROM sqlite_master }
367: catch { db eval { PRAGMA journal_mode = WAL } }
368: db eval {
369: BEGIN;
370: INSERT INTO t1 VALUES(NULL, randomblob(110+$E(pid)));
371: }
372: incr nRun 1
373: incr nDel [file exists test.db-journal]
374: if {[file exists test.db-journal] + [file exists test.db-wal] != 1} {
375: error "File-system looks bad..."
376: }
377: db eval COMMIT
378:
379: integrity_check
380: db close
381: }
382: set {} "[expr $nRun-$nDel] w, $nDel r"
383: }
384:
385: do_thread_test walthread-3 -seconds $seconds(walthread-3) -init {
386: execsql {
387: PRAGMA journal_mode = WAL;
388: CREATE TABLE t1(cnt PRIMARY KEY, sum1, sum2);
389: CREATE INDEX i1 ON t1(sum1);
390: CREATE INDEX i2 ON t1(sum2);
391: INSERT INTO t1 VALUES(0, 0, 0);
392: }
393: } -thread t 10 {
394:
395: set nextwrite $E(pid)
396:
397: proc wal_hook {zDb nEntry} {
398: if {$nEntry>10} {
399: set rc [catch { db eval {PRAGMA wal_checkpoint} } msg]
400: if {$rc && $msg != "database is locked"} { error $msg }
401: }
402: return 0
403: }
404: db wal_hook wal_hook
405:
406: while {[tt_continue]} {
407: set max 0
408: while { $max != ($nextwrite-1) && [tt_continue] } {
409: set max [db eval { SELECT max(cnt) FROM t1 }]
410: }
411:
412: if {[tt_continue]} {
413: set sum1 [db eval { SELECT sum(cnt) FROM t1 }]
414: set sum2 [db eval { SELECT sum(sum1) FROM t1 }]
415: db eval { INSERT INTO t1 VALUES($nextwrite, $sum1, $sum2) }
416: incr nextwrite $E(nthread)
417: integrity_check
418: }
419: }
420:
421: set {} ok
422: } -check {
423: puts " Final db contains [db eval {SELECT count(*) FROM t1}] rows"
424: puts " Final integrity-check says: [db eval {PRAGMA integrity_check}]"
425:
426: # Check that the contents of the database are Ok.
427: set c 0
428: set s1 0
429: set s2 0
430: db eval { SELECT cnt, sum1, sum2 FROM t1 ORDER BY cnt } {
431: if {$c != $cnt || $s1 != $sum1 || $s2 != $sum2} {
432: error "database content is invalid"
433: }
434: incr s2 $s1
435: incr s1 $c
436: incr c 1
437: }
438: }
439:
440: do_thread_test2 walthread-4 -seconds $seconds(walthread-4) -init {
441: execsql {
442: PRAGMA journal_mode = WAL;
443: CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE);
444: }
445: } -thread r 1 {
446: # This connection only ever reads the database. Therefore the
447: # busy-handler is not required. Disable it to check that this is true.
448: #
449: # UPDATE: That is no longer entirely true - as we don't use a blocking
450: # lock to enter RECOVER state. Which means there is a small chance a
451: # reader can see an SQLITE_BUSY.
452: #
453: while {[tt_continue]} {
454: integrity_check
455: }
456: set {} ok
457: } -thread w 1 {
458:
459: proc wal_hook {zDb nEntry} {
460: if {$nEntry>15} {db eval {PRAGMA wal_checkpoint}}
461: return 0
462: }
463: db wal_hook wal_hook
464: set row 1
465: while {[tt_continue]} {
466: db eval { REPLACE INTO t1 VALUES($row, randomblob(300)) }
467: incr row
468: if {$row == 10} { set row 1 }
469: }
470:
471: set {} ok
472: }
473:
474:
475: # This test case attempts to provoke a deadlock condition that existed in
476: # the unix VFS at one point. The problem occurred only while recovering a
477: # very large wal file (one that requires a wal-index larger than the
478: # initial default allocation of 64KB).
479: #
480: do_thread_test walthread-5 -seconds $seconds(walthread-5) -init {
481:
482: proc log_file_size {nFrame pgsz} {
483: expr {12 + ($pgsz+16)*$nFrame}
484: }
485:
486: execsql {
487: PRAGMA page_size = 1024;
488: PRAGMA journal_mode = WAL;
489: CREATE TABLE t1(x);
490: BEGIN;
491: INSERT INTO t1 VALUES(randomblob(900));
492: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */
493: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */
494: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */
495: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */
496: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */
497: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */
498: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */
499: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */
500: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */
501: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */
502: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */
503: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */
504: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */
505: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */
506: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32768 */
507: INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 65536 */
508: COMMIT;
509: }
510:
511: forcecopy test.db-wal bak.db-wal
512: forcecopy test.db bak.db
513: db close
514:
515: forcecopy bak.db-wal test.db-wal
516: forcecopy bak.db test.db
517:
518: if {[file size test.db-wal] < [log_file_size [expr 64*1024] 1024]} {
519: error "Somehow failed to create a large log file"
520: }
521: puts "Database with large log file recovered. Now running clients..."
522: } -thread T 5 {
523: db eval { SELECT count(*) FROM t1 }
524: }
525: unset -nocomplain seconds
526:
527: finish_test
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to walthread.test CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / sqlite3 / test