AryaWu/sqlite
0
1# 2009 December 032#3# May you do good and not evil.4# May you find forgiveness for yourself and forgive others.5# May you share freely, never taking more than you give.6#7#***********************************************************************8#9# The tests in this file are structural coverage tests for FTS3.10#11 12set testdir [file dirname $argv0]13source $testdir/tester.tcl14 15# If this build does not include FTS3, skip the tests in this file.16#17ifcapable !fts3 { finish_test ; return }18source $testdir/fts3_common.tcl19source $testdir/malloc_common.tcl20 21set DO_MALLOC_TEST 022set testprefix fts3cov23 24#--------------------------------------------------------------------------25# When it first needs to read a block from the %_segments table, the FTS3 26# module compiles an SQL statement for that purpose. The statement is 27# stored and reused each subsequent time a block is read. This test case 28# tests the effects of an OOM error occurring while compiling the statement.29#30# Similarly, when FTS3 first needs to scan through a set of segment leaves31# to find a set of documents that matches a term, it allocates a string32# containing the text of the required SQL, and compiles one or more 33# statements to traverse the leaves. This test case tests that OOM errors34# that occur while allocating this string and statement are handled correctly35# also.36#37do_test fts3cov-1.1 {38 execsql { 39 CREATE VIRTUAL TABLE t1 USING fts3(x);40 INSERT INTO t1(t1) VALUES('nodesize=24');41 BEGIN;42 INSERT INTO t1 VALUES('Is the night chilly and dark?');43 INSERT INTO t1 VALUES('The night is chilly, but not dark.');44 INSERT INTO t1 VALUES('The thin gray cloud is spread on high,');45 INSERT INTO t1 VALUES('It covers but not hides the sky.');46 COMMIT;47 SELECT count(*)>0 FROM t1_segments;48 }49} {1}50 51set DO_MALLOC_TEST 152do_restart_select_test fts3cov-1.2 {53 SELECT docid FROM t1 WHERE t1 MATCH 'chilly';54} {1 2}55set DO_MALLOC_TEST 056 57#--------------------------------------------------------------------------58# When querying the full-text index, if an expected internal node block is 59# missing from the %_segments table, or if a NULL value is stored in the 60# %_segments table instead of a binary blob, database corruption should be 61# reported.62#63# Even with tiny 24 byte nodes, it takes a fair bit of data to produce a64# segment b-tree that uses the %_segments table to store internal nodes. 65#66do_test fts3cov-2.1 {67 execsql {68 INSERT INTO t1(t1) VALUES('nodesize=24');69 BEGIN;70 INSERT INTO t1 VALUES('The moon is behind, and at the full;');71 INSERT INTO t1 VALUES('And yet she looks both small and dull.');72 INSERT INTO t1 VALUES('The night is chill, the cloud is gray:');73 INSERT INTO t1 VALUES('''T is a month before the month of May,');74 INSERT INTO t1 VALUES('And the Spring comes slowly up this way.');75 INSERT INTO t1 VALUES('The lovely lady, Christabel,');76 INSERT INTO t1 VALUES('Whom her father loves so well,');77 INSERT INTO t1 VALUES('What makes her in the wood so late,');78 INSERT INTO t1 VALUES('A furlong from the castle gate?');79 INSERT INTO t1 VALUES('She had dreams all yesternight');80 INSERT INTO t1 VALUES('Of her own betrothed knight;');81 INSERT INTO t1 VALUES('And she in the midnight wood will pray');82 INSERT INTO t1 VALUES('For the weal of her lover that''s far away.');83 COMMIT;84 }85 execsql {86 INSERT INTO t1(t1) VALUES('optimize');87 SELECT substr(hex(root), 1, 2) FROM t1_segdir;88 }89} {03}90 91# Test the "missing entry" case:92sqlite3_db_config db DEFENSIVE 093do_test fts3cov-2.2 {94 set root [db one {SELECT root FROM t1_segdir}]95 read_fts3varint [string range $root 1 end] left_child96 execsql { DELETE FROM t1_segments WHERE blockid = $left_child }97} {}98do_error_test fts3cov-2.3 {99 SELECT * FROM t1 WHERE t1 MATCH 'c*'100} {database disk image is malformed}101 102# Test the "replaced with NULL" case:103do_test fts3cov-2.4 {104 execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }105} {}106do_error_test fts3cov-2.5 {107 SELECT * FROM t1 WHERE t1 MATCH 'cloud'108} {database disk image is malformed}109 110#--------------------------------------------------------------------------111# The following tests are to test the effects of OOM errors while storing112# terms in the pending-hash table. Specifically, while creating doclist113# blobs to store in the table. More specifically, to test OOM errors while114# appending column numbers to doclists. For example, if a doclist consists115# of:116#117# <docid> <column 0 offset-list> 0x01 <column N> <column N offset-list>118#119# The following tests check that malloc errors encountered while appending120# the "0x01 <column N>" data to the dynamically growable blob used to 121# accumulate the doclist in memory are handled correctly.122#123do_test fts3cov-3.1 {124 set cols [list]125 set vals [list]126 for {set i 0} {$i < 120} {incr i} {127 lappend cols "col$i"128 lappend vals "'word'"129 }130 execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"131} {}132set DO_MALLOC_TEST 1 133do_write_test fts3cov-3.2 t2_content "134 INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,])135"136do_write_test fts3cov-3.3 t2_content "137 INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,])138"139do_write_test fts3cov-3.4 t2_content "140 INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,])141"142 143#-------------------------------------------------------------------------144# If too much data accumulates in the pending-terms hash table, it is145# flushed to the database automatically, even if the transaction has not146# finished. The following tests check the effects of encountering an OOM 147# while doing this.148#149do_test fts3cov-4.1 {150 execsql {151 CREATE VIRTUAL TABLE t3 USING fts3(x);152 INSERT INTO t3(t3) VALUES('nodesize=24');153 INSERT INTO t3(t3) VALUES('maxpending=100');154 }155} {}156set DO_MALLOC_TEST 1 157do_write_test fts3cov-4.2 t3_content {158 INSERT INTO t3(docid, x)159 SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL160 SELECT 3, 'And comforted fair Geraldine:' UNION ALL161 SELECT 4, '''O well, bright dame, may you command' UNION ALL162 SELECT 5, 'The service of Sir Leoline;' UNION ALL163 SELECT 2, 'And gladly our stout chivalry' UNION ALL164 SELECT 7, 'Will he send forth, and friends withal,' UNION ALL165 SELECT 8, 'To guide and guard you safe and free' UNION ALL166 SELECT 6, 'Home to your noble father''s hall.'''167}168 169#-------------------------------------------------------------------------170# When building the internal tree structure for each segment b-tree, FTS3171# assumes that the content of each internal node will be less than172# $nodesize bytes, where $nodesize is the advisory node size. If this turns173# out to be untrue, then an extra buffer must be malloc'd for each term.174# This test case tests these paths and the effects of said mallocs failing175# by inserting insert a document with some fairly large terms into a176# full-text table with a very small node-size. 177#178# Test this handling of large terms in three contexts:179#180# 1. When flushing the pending-terms table.181# 2. When optimizing the data structures using the INSERT syntax. 182# 2. When optimizing the data structures using the deprecated SELECT syntax. 183#184do_test fts3cov-5.1 {185 execsql {186 CREATE VIRTUAL TABLE t4 USING fts3(x);187 INSERT INTO t4(t4) VALUES('nodesize=24');188 }189} {}190set DO_MALLOC_TEST 1191 192# Test when flushing pending-terms table.193do_write_test fts3cov-5.2 t4_content {194 INSERT INTO t4195 SELECT 'ItisanancientMarinerAndhestoppethoneofthreeAA' UNION ALL196 SELECT 'ItisanancientMarinerAndhestoppethoneofthreeBB' UNION ALL197 SELECT 'ItisanancientMarinerAndhestoppethoneofthreeCC' UNION ALL198 SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstAA' UNION ALL199 SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstBB' UNION ALL200 SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstCC'201}202 203# Test when optimizing via INSERT.204do_test fts3cov-5.3 { execsql { INSERT INTO t4 VALUES('extra!') } } {}205do_write_test fts3cov-5.2 t4_segments { INSERT INTO t4(t4) VALUES('optimize') }206 207# Test when optimizing via SELECT.208do_test fts3cov-5.5 { execsql { INSERT INTO t4 VALUES('more extra!') } } {}209do_write_test fts3cov-5.6 t4_segments {210 SELECT * FROM (SELECT optimize(t4) FROM t4 LIMIT 1)211 EXCEPT SELECT 'Index optimized'212}213 214#-------------------------------------------------------------------------215# When merging all segments at a given level to create a single segment216# at level+1, FTS3 runs a query of the form:217#218# SELECT count(*) FROM %_segdir WHERE level = ?219#220# The query is compiled the first time this operation is required and221# reused thereafter. This test aims to test the effects of an OOM while222# preparing and executing this query for the first time.223#224# Then, keep inserting rows into the table so that the effects of an OOM225# while re-executing the same query can also be tested.226#227do_test fts3cov-6.1 {228 execsql { CREATE VIRTUAL TABLE t5 USING fts3(x) }229 for {set i 0} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }230 execsql { SELECT count(*) FROM t5_segdir }231} {16}232 233# First time.234db close235sqlite3 db test.db236do_write_test fts3cov-6.2 t5_content {237 INSERT INTO t5 VALUES('segment number 16!');238}239 240# Second time.241do_test fts3cov-6.3 {242 for {set i 1} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }243 execsql { SELECT count(*) FROM t5_segdir }244} {17}245do_write_test fts3cov-6.4 t5_content {246 INSERT INTO t5 VALUES('segment number 16!');247}248 249#-------------------------------------------------------------------------250# Update the docid of a row. Test this in two scenarios:251#252# 1. When the row being updated is the only row in the table.253# 2. When it is not.254#255# The two cases above take different paths because in case 1 all data 256# structures can simply be emptied before inserting the new row record.257# In case 2, the data structures actually have to be updated.258#259do_test fts3cov-7.1 {260 execsql {261 CREATE VIRTUAL TABLE t7 USING fts3(a, b, c);262 INSERT INTO t7 VALUES('A', 'B', 'C');263 UPDATE t7 SET docid = 5;264 SELECT docid, * FROM t7;265 }266} {5 A B C}267do_test fts3cov-7.2 {268 execsql {269 INSERT INTO t7 VALUES('D', 'E', 'F');270 UPDATE t7 SET docid = 1 WHERE docid = 6;271 SELECT docid, * FROM t7;272 }273} {1 D E F 5 A B C}274 275#-------------------------------------------------------------------------276# If a set of documents are modified within a transaction, the 277# pending-terms table must be flushed each time a document with a docid278# less than or equal to the previous docid is modified. 279#280# This test checks the effects of an OOM error occurring when the 281# pending-terms table is flushed for this reason as part of a DELETE 282# statement.283#284do_malloc_test fts3cov-8 -sqlprep {285 BEGIN;286 CREATE VIRTUAL TABLE t8 USING fts3;287 INSERT INTO t8 VALUES('the output of each batch run');288 INSERT INTO t8 VALUES('(possibly a day''s work)');289 INSERT INTO t8 VALUES('was written to two separate disks');290 COMMIT;291} -sqlbody {292 BEGIN;293 DELETE FROM t8 WHERE rowid = 3;294 DELETE FROM t8 WHERE rowid = 2;295 DELETE FROM t8 WHERE rowid = 1;296 COMMIT;297}298 299#-------------------------------------------------------------------------300# Test some branches in the code that handles "special" inserts like:301#302# INSERT INTO t1(t1) VALUES('optimize');303#304# Also test that an optimize (INSERT method) works on an empty table.305#306set DO_MALLOC_TEST 0307do_test fts3cov-9.1 {308 execsql { CREATE VIRTUAL TABLE xx USING fts3 }309} {}310do_error_test fts3cov-9.2 {311 INSERT INTO xx(xx) VALUES('optimise'); -- British spelling312} {SQL logic error}313do_error_test fts3cov-9.3 {314 INSERT INTO xx(xx) VALUES('short');315} {SQL logic error}316do_error_test fts3cov-9.4 {317 INSERT INTO xx(xx) VALUES('waytoolongtobecorrect');318} {SQL logic error}319do_test fts3cov-9.5 {320 execsql { INSERT INTO xx(xx) VALUES('optimize') }321} {}322 323#-------------------------------------------------------------------------324# Test that a table can be optimized in the middle of a transaction when325# the pending-terms table is non-empty. This case involves some extra326# branches because data must be read not only from the database, but327# also from the pending-terms table.328#329do_malloc_test fts3cov-10 -sqlprep {330 CREATE VIRTUAL TABLE t10 USING fts3;331 INSERT INTO t10 VALUES('Optimising images for the web is a tricky business');332 BEGIN;333 INSERT INTO t10 VALUES('You have to get the right balance between');334} -sqlbody {335 INSERT INTO t10(t10) VALUES('optimize');336}337 338#-------------------------------------------------------------------------339# Test a full-text query for a term that was once in the index, but is340# no longer.341#342do_test fts3cov-11.1 {343 execsql { 344 CREATE VIRTUAL TABLE xx USING fts3;345 INSERT INTO xx VALUES('one two three');346 INSERT INTO xx VALUES('four five six');347 DELETE FROM xx WHERE docid = 1;348 }349 execsql { SELECT * FROM xx WHERE xx MATCH 'two' }350} {}351 352 353do_malloc_test fts3cov-12 -sqlprep {354 CREATE VIRTUAL TABLE t12 USING fts3;355 INSERT INTO t12 VALUES('is one of the two togther');356 BEGIN;357 INSERT INTO t12 VALUES('one which was appropriate at the time');358} -sqlbody {359 SELECT * FROM t12 WHERE t12 MATCH 'one'360}361 362do_malloc_test fts3cov-13 -sqlprep {363 PRAGMA encoding = 'UTF-16';364 CREATE VIRTUAL TABLE t13 USING fts3;365 INSERT INTO t13 VALUES('two scalar functions');366 INSERT INTO t13 VALUES('scalar two functions');367 INSERT INTO t13 VALUES('functions scalar two');368} -sqlbody {369 SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two';370 SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two';371 SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two';372}373 374do_execsql_test 14.0 {375 CREATE VIRTUAL TABLE t14 USING fts4(a, b);376 INSERT INTO t14 VALUES('one two three', 'one three four');377 INSERT INTO t14 VALUES('a b c', 'd e a');378}379do_execsql_test 14.1 {380 SELECT rowid FROM t14 WHERE t14 MATCH '"one two three"'381} {1}382do_execsql_test 14.2 {383 SELECT rowid FROM t14 WHERE t14 MATCH '"one four"'384} {}385do_execsql_test 14.3 {386 SELECT rowid FROM t14 WHERE t14 MATCH '"e a"'387} {2}388do_execsql_test 14.5 {389 SELECT rowid FROM t14 WHERE t14 MATCH '"e b"'390} {}391do_catchsql_test 14.6 {392 SELECT rowid FROM t14 WHERE rowid MATCH 'one'393} {1 {unable to use function MATCH in the requested context}}394do_catchsql_test 14.7 {395 SELECT rowid FROM t14 WHERE docid MATCH 'one'396} {1 {unable to use function MATCH in the requested context}}397 398do_execsql_test 15.0 {399 CREATE VIRTUAL TABLE t15 USING fts4(a, b, c);400 INSERT INTO t15 VALUES('abc def ghi', 'abc2 def2 ghi2', 'abc3 def3 ghi3');401 INSERT INTO t15 VALUES('abc2 def2 ghi2', 'abc2 def2 ghi2', 'abc def3 ghi3');402}403do_execsql_test 15.1 {404 SELECT rowid FROM t15 WHERE t15 MATCH '"abc* def2"'405} {1 2}406 407# Test a corruption case.408#409sqlite3_db_config db DEFENSIVE 0410do_execsql_test 16.1 {411 CREATE VIRTUAL TABLE t16 USING fts4;412 INSERT INTO t16 VALUES('theoretical work to examine the relationship');413 INSERT INTO t16 VALUES('solution of our problems on the invisible');414 DELETE FROM t16_content WHERE rowid = 2;415}416do_catchsql_test 16.2 {417 SELECT * FROM t16 WHERE t16 MATCH 'invisible'418} {1 {database disk image is malformed}}419 420# And another corruption test case.421#422do_execsql_test 17.1 {423 CREATE VIRTUAL TABLE t17 USING fts4;424 INSERT INTO t17(content) VALUES('one one one');425 UPDATE t17_segdir SET root = X'00036F6E65FFFFFFFFFFFFFFFFFFFFFF02030300'426} {}427do_catchsql_test 17.2 {428 SELECT * FROM t17 WHERE t17 MATCH 'one'429} {1 {database disk image is malformed}}430 431 432 433 434finish_test435