AryaWu/sqlite
0
1# 2010 November 022#3# The author disclaims copyright to this source code. In place of4# a legal notice, here is a blessing:5#6# May you do good and not evil.7# May you find forgiveness for yourself and forgive others.8# May you share freely, never taking more than you give.9#10#***********************************************************************11# This file implements regression tests for the FTS3 module. The focus12# of this file is tables created with the "matchinfo=fts3" option.13#14 15set testdir [file dirname $argv0]16source $testdir/tester.tcl17 18# If SQLITE_ENABLE_FTS3 is not defined, omit this file.19ifcapable !fts3 { finish_test ; return }20 21set testprefix fts3matchinfo22set sqlite_fts3_enable_parentheses 023 24proc mit {blob} {25 set scan(littleEndian) i*26 set scan(bigEndian) I*27 binary scan $blob $scan($::tcl_platform(byteOrder)) r28 return $r29}30db func mit mit31 32do_execsql_test 1.0 {33 CREATE VIRTUAL TABLE t1 USING fts4(matchinfo=fts3);34 SELECT name FROM sqlite_master WHERE type = 'table';35} {t1 t1_content t1_segments t1_segdir t1_stat}36 37do_execsql_test 1.1 {38 INSERT INTO t1(content) VALUES('I wandered lonely as a cloud');39 INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,');40 INSERT INTO t1(content) VALUES('When all at once I saw a crowd,');41 INSERT INTO t1(content) VALUES('A host, of golden daffodils,');42 SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I';43} {{1 1 1 2 2} {1 1 1 2 2}}44 45# Now create an FTS4 table that does not specify matchinfo=fts3.46#47do_execsql_test 1.2 {48 CREATE VIRTUAL TABLE t2 USING fts4;49 INSERT INTO t2 SELECT * FROM t1;50 SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';51} {{1 1 1 2 2} {1 1 1 2 2}}52 53# Test some syntax-error handling.54#55do_catchsql_test 2.0 {56 CREATE VIRTUAL TABLE x1 USING fts4(matchinfo=fs3);57} {1 {unrecognized matchinfo: fs3}}58do_catchsql_test 2.1 {59 CREATE VIRTUAL TABLE x2 USING fts4(mtchinfo=fts3);60} {1 {unrecognized parameter: mtchinfo=fts3}}61do_catchsql_test 2.2 {62 CREATE VIRTUAL TABLE x2 USING fts4(matchinfo=fts5);63} {1 {unrecognized matchinfo: fts5}}64 65# Check that with fts3, the "=" character is permitted in column definitions.66#67do_execsql_test 3.1 {68 CREATE VIRTUAL TABLE t3 USING fts3(mtchinfo=fts3);69 INSERT INTO t3(mtchinfo) VALUES('Beside the lake, beneath the trees');70 SELECT mtchinfo FROM t3;71} {{Beside the lake, beneath the trees}}72 73do_execsql_test 3.2 {74 CREATE VIRTUAL TABLE xx USING FTS4;75}76do_execsql_test 3.3 {77 SELECT * FROM xx WHERE xx MATCH 'abc';78}79do_execsql_test 3.4 {80 SELECT * FROM xx WHERE xx MATCH 'a b c';81}82 83 84#--------------------------------------------------------------------------85# Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function.86#87# The first argument - $tn - is a test identifier. This may be either a88# full identifier (i.e. "fts3matchinfo-1.1") or, if global var $testprefix89# is set, just the numeric component (i.e. "1.1").90#91# The second argument is the name of an FTSX table. The third is the 92# full text of a WHERE/MATCH expression to query the table for 93# (i.e. "t1 MATCH 'abc'"). The final argument - $results - should be a94# key-value list (serialized array) with matchinfo() format specifiers95# as keys, and the results of executing the statement:96#97# SELECT matchinfo($tbl, '$key') FROM $tbl WHERE $expr98#99# For example:100#101# CREATE VIRTUAL TABLE t1 USING fts4;102# INSERT INTO t1 VALUES('abc');103# INSERT INTO t1 VALUES('def');104# INSERT INTO t1 VALUES('abc abc');105#106# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {107# n {3 3}108# p {1 1}109# c {1 1}110# x {{1 3 2} {2 3 2}}111# }112#113# If the $results list contains keys mapped to "-" instead of a matchinfo()114# result, then this command computes the expected results based on other115# mappings to test the matchinfo() function. For example, the command above116# could be changed to:117#118# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {119# n {3 3} p {1 1} c {1 1} x {{1 3 2} {2 3 2}}120# pcx -121# }122#123# And this command would compute the expected results for matchinfo(t1, 'pcx')124# based on the results of matchinfo(t1, 'p'), matchinfo(t1, 'c') and 125# matchinfo(t1, 'x') in order to test 'pcx'.126#127proc do_matchinfo_test {tn tbl expr results} {128 129 foreach {fmt res} $results {130 if {$res == "-"} continue131 set resarray($fmt) $res132 }133 134 set nRow 0135 foreach {fmt res} [array get resarray] {136 if {[llength $res]>$nRow} { set nRow [llength $res] }137 }138 139 # Construct expected results for any formats for which the caller 140 # supplied result is "-".141 #142 foreach {fmt res} $results {143 if {$res == "-"} {144 set res [list]145 for {set iRow 0} {$iRow<$nRow} {incr iRow} {146 set rowres [list]147 foreach c [split $fmt ""] {148 set rowres [concat $rowres [lindex $resarray($c) $iRow]]149 }150 lappend res $rowres151 }152 set resarray($fmt) $res153 }154 }155 156 # Test each matchinfo() request individually.157 #158 foreach {fmt res} [array get resarray] {159 set sql "SELECT mit(matchinfo($tbl, '$fmt')) FROM $tbl WHERE $expr"160 do_execsql_test $tn.$fmt $sql [normalize2 $res]161 }162 163 # Test them all executed together (multiple invocations of matchinfo()).164 #165 set exprlist [list]166 foreach {format res} [array get resarray] {167 lappend exprlist "mit(matchinfo($tbl, '$format'))"168 }169 set allres [list]170 for {set iRow 0} {$iRow<$nRow} {incr iRow} {171 foreach {format res} [array get resarray] {172 lappend allres [lindex $res $iRow]173 }174 }175 set sql "SELECT [join $exprlist ,] FROM $tbl WHERE $expr"176 do_execsql_test $tn.multi $sql [normalize2 $allres]177}178proc normalize2 {list_of_lists} {179 set res [list]180 foreach elem $list_of_lists {181 lappend res [list {*}$elem]182 }183 return $res184}185 186 187do_execsql_test 4.1.0 {188 CREATE VIRTUAL TABLE t4 USING fts4(x, y);189 INSERT INTO t4 VALUES('a b c d e', 'f g h i j');190 INSERT INTO t4 VALUES('f g h i j', 'a b c d e');191}192 193do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {194 p {3 3}195 c {2 2}196 x {197 {1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 0 1 1}198 {0 1 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1}199 }200 n {2 2}201 l {{5 5} {5 5}}202 a {{5 5} {5 5}}203 204 s {{3 0} {0 3}}205 206 xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -207 xpxsscplax -208}209 210do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {211 p {1 1}212 c {2 2}213 x {214 {0 1 1 1 1 1}215 {1 1 1 0 1 1}216 }217 n {2 2}218 l {{5 5} {5 5}}219 a {{5 5} {5 5}}220 221 s {{0 1} {1 0}}222 223 xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -224 sxsxs -225}226 227do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} }228do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }229do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }230do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} }231do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {232 x { 233 {0 1 1 1 1 1 0 0 0 0 0 0} 234 {1 1 1 0 1 1 0 0 0 0 0 0}235 }236}237do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f -abcd'} {238 x { 239 {0 1 1 1 1 1}240 {1 1 1 0 1 1}241 }242}243 244do_execsql_test 4.2.0 {245 CREATE VIRTUAL TABLE t5 USING fts4;246 INSERT INTO t5 VALUES('a a a a a');247 INSERT INTO t5 VALUES('a b a b a');248 INSERT INTO t5 VALUES('c b c b c');249 INSERT INTO t5 VALUES('x x x x x');250}251do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'} { 252 x {{5 8 2 5 8 2} {3 8 2 3 8 2}}253 s {2 1} 254}255do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'} { s {2} }256do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'} { s {3} }257do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'} { s {3 1} }258do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }259do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} }260 261do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";262 263# It used to be that the second 'a' token would be deferred. That doesn't264# work any longer.265if 0 {266 do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { 267 x {{5 8 2 5 5 5} {3 8 2 3 5 5}}268 s {2 1} 269 }270}271 272do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} }273do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} }274do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} }275do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }276do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} }277 278do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') }279 280ifcapable fts4_deferred {281 sqlite3_db_config db DEFENSIVE 0282 do_execsql_test 4.4.0.2 {283 UPDATE t5_segments 284 SET block = zeroblob(length(block)) 285 WHERE length(block)>10000;286 }287}288 289do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} }290do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} }291do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} }292do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'} { s {3} }293do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'} { s {3 1} }294do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }295 296do_execsql_test 4.5.0 {297 CREATE VIRTUAL TABLE t6 USING fts4(a, b, c);298 INSERT INTO t6 VALUES('a', 'b', 'c');299}300do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} }301 302 303#-------------------------------------------------------------------------304# Check the following restrictions:305#306# + Matchinfo flags 'a', 'l' and 'n' can only be used with fts4, not fts3.307# + Matchinfo flag 'l' cannot be used with matchinfo=fts3.308#309do_execsql_test 5.1 {310 CREATE VIRTUAL TABLE t7 USING fts3(a, b);311 INSERT INTO t7 VALUES('u v w', 'x y z');312 313 CREATE VIRTUAL TABLE t8 USING fts4(a, b, matchinfo=fts3);314 INSERT INTO t8 VALUES('u v w', 'x y z');315}316 317do_catchsql_test 5.2.1 { 318 SELECT matchinfo(t7, 'a') FROM t7 WHERE t7 MATCH 'x y'319} {1 {unrecognized matchinfo request: a}}320do_catchsql_test 5.2.2 { 321 SELECT matchinfo(t7, 'l') FROM t7 WHERE t7 MATCH 'x y'322} {1 {unrecognized matchinfo request: l}}323do_catchsql_test 5.2.3 { 324 SELECT matchinfo(t7, 'n') FROM t7 WHERE t7 MATCH 'x y'325} {1 {unrecognized matchinfo request: n}}326 327do_catchsql_test 5.3.1 { 328 SELECT matchinfo(t8, 'l') FROM t8 WHERE t8 MATCH 'x y'329} {1 {unrecognized matchinfo request: l}}330 331#-------------------------------------------------------------------------332# Test that the offsets() function handles corruption in the %_content333# table correctly.334#335do_execsql_test 6.1 {336 CREATE VIRTUAL TABLE t9 USING fts4;337 INSERT INTO t9 VALUES(338 'this record is used to try to dectect corruption'339 );340 SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';341} {{0 0 20 2 0 0 27 2}}342 343sqlite3_db_config db DEFENSIVE 0344do_catchsql_test 6.2 {345 UPDATE t9_content SET c0content = 'this record is used to'; 346 SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';347} {1 {database disk image is malformed}}348 349#-------------------------------------------------------------------------350# Test the outcome of matchinfo() when used within a query that does not351# use the full-text index (i.e. lookup by rowid or full-table scan).352#353do_execsql_test 7.1 {354 CREATE VIRTUAL TABLE t10 USING fts4;355 INSERT INTO t10 VALUES('first record');356 INSERT INTO t10 VALUES('second record');357}358do_execsql_test 7.2 {359 SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10;360} {blob 0 blob 0}361do_execsql_test 7.3 {362 SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10 WHERE docid=1;363} {blob 0}364do_execsql_test 7.4 {365 SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) 366 FROM t10 WHERE t10 MATCH 'record'367} {blob 20 blob 20}368 369#-------------------------------------------------------------------------370# Test a special case - matchinfo('nxa') with many zero length documents. 371# Special because "x" internally uses a statement used by both "n" and "a". 372# This was causing a problem at one point in the obscure case where the373# total number of bytes of data stored in an fts3 table was greater than374# the number of rows. i.e. when the following query returns true:375#376# SELECT sum(length(content)) < count(*) FROM fts4table;377#378do_execsql_test 8.1 {379 CREATE VIRTUAL TABLE t11 USING fts4;380 INSERT INTO t11(t11) VALUES('nodesize=24');381 INSERT INTO t11 VALUES('quitealongstringoftext');382 INSERT INTO t11 VALUES('anotherquitealongstringoftext');383 INSERT INTO t11 VALUES('athirdlongstringoftext');384 INSERT INTO t11 VALUES('andonemoreforgoodluck');385}386do_test 8.2 {387 for {set i 0} {$i < 200} {incr i} {388 execsql { INSERT INTO t11 VALUES('') }389 }390 execsql { INSERT INTO t11(t11) VALUES('optimize') }391} {}392do_execsql_test 8.3 {393 SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'394} {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}}395 396# Corruption related tests.397sqlite3_db_config db DEFENSIVE 0398do_execsql_test 8.4.1.1 { UPDATE t11_stat SET value = X'0000'; }399do_catchsql_test 8.5.1.2 {400 SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'401} {1 {database disk image is malformed}}402 403do_execsql_test 8.4.2.1 { UPDATE t11_stat SET value = X'00'; }404do_catchsql_test 8.5.2.2 {405 SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'406} {1 {database disk image is malformed}}407 408do_execsql_test 8.4.3.1 { UPDATE t11_stat SET value = NULL; }409do_catchsql_test 8.5.3.2 {410 SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'411} {1 {database disk image is malformed}}412 413#-------------------------------------------------------------------------414do_execsql_test 8.1 {415 CREATE VIRTUAL TABLE t12 USING fts4;416 INSERT INTO t12 VALUES('a b c d');417 SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'a NEAR/1 d OR a';418} {{0 0 0 0 0 0 1 1 1}}419do_execsql_test 8.2 {420 INSERT INTO t12 VALUES('a d c d');421 SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'a NEAR/1 d OR a';422} {423 {0 1 1 0 1 1 1 2 2} {1 1 1 1 1 1 1 2 2}424}425do_execsql_test 8.3 {426 INSERT INTO t12 VALUES('a d d a');427 SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'a NEAR/1 d OR a';428} {429 {0 3 2 0 3 2 1 4 3} {1 3 2 1 3 2 1 4 3} {2 3 2 2 3 2 2 4 3}430}431 432do_execsql_test 9.1 {433 CREATE VIRTUAL TABLE ft2 USING fts4;434 INSERT INTO ft2 VALUES('a b c d e');435 INSERT INTO ft2 VALUES('f a b c d');436 SELECT snippet(ft2, '[', ']', '', -1, 1) FROM ft2 WHERE ft2 MATCH 'c';437} {{[c]} {[c]}}438 439#---------------------------------------------------------------------------440# Test for a memory leak441#442do_execsql_test 10.1 {443 DROP TABLE t10;444 CREATE VIRTUAL TABLE t10 USING fts4(idx, value);445 INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');446 SELECT docId, t10.*447 FROM t10448 JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x449 WHERE t10 MATCH x.idx450 AND matchinfo(t10) not null451 GROUP BY docId452 ORDER BY 1;453} {1 1 one 2 2 two 3 3 three}454 455#---------------------------------------------------------------------------456# Test the 'y' matchinfo flag457#458set sqlite_fts3_enable_parentheses 1459reset_db460do_execsql_test 11.0 {461 CREATE VIRTUAL TABLE tt USING fts3(x, y);462 INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1463 INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2464 INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3465 INSERT INTO tt VALUES('a c f f g d', 'd b f d e g'); -- 4466 INSERT INTO tt VALUES('g a c f c f', 'd g g b c c'); -- 5467 INSERT INTO tt VALUES('g a c e b b', 'd b f b g g'); -- 6468 INSERT INTO tt VALUES('f d a a f c', 'e e a d c f'); -- 7469 INSERT INTO tt VALUES('a c b b g f', 'a b a e d f'); -- 8470 INSERT INTO tt VALUES('b a f e c c', 'f d b b a b'); -- 9471 INSERT INTO tt VALUES('f d c e a c', 'f a f a a f'); -- 10472}473 474db func mit mit475foreach {tn expr res} {476 1 "a" {477 1 {1 2} 2 {1 0} 3 {0 1} 4 {1 0} 5 {1 0}478 6 {1 0} 7 {2 1} 8 {1 2} 9 {1 1} 10 {1 3}479 }480 481 2 "b" {482 1 {0 1} 2 {1 0} 3 {1 2} 4 {0 1} 5 {0 1}483 6 {2 2} 8 {2 1} 9 {1 3} 484 }485 486 3 "y:a" {487 1 {0 2} 3 {0 1} 488 7 {0 1} 8 {0 2} 9 {0 1} 10 {0 3}489 }490 491 4 "x:a" {492 1 {1 0} 2 {1 0} 4 {1 0} 5 {1 0}493 6 {1 0} 7 {2 0} 8 {1 0} 9 {1 0} 10 {1 0}494 }495 496 5 "a OR b" {497 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1}498 6 {1 0 2 2} 7 {2 1 0 0} 8 {1 2 2 1} 9 {1 1 1 3} 10 {1 3 0 0}499 }500 501 6 "a AND b" {502 1 {1 2 0 1} 2 {1 0 1 0} 3 {0 1 1 2} 4 {1 0 0 1} 5 {1 0 0 1}503 6 {1 0 2 2} 8 {1 2 2 1} 9 {1 1 1 3} 504 }505 506 7 "a OR (a AND b)" {507 1 {1 2 1 2 0 1} 2 {1 0 1 0 1 0} 3 {0 1 0 1 1 2} 4 {1 0 1 0 0 1} 508 5 {1 0 1 0 0 1} 6 {1 0 1 0 2 2} 7 {2 1 0 0 0 0} 8 {1 2 1 2 2 1} 509 9 {1 1 1 1 1 3} 10 {1 3 0 0 0 0}510 }511 512} {513 do_execsql_test 11.1.$tn.1 {514 SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr515 } $res516 517 set r2 [list]518 foreach {rowid L} $res {519 lappend r2 $rowid520 set M [list]521 foreach {a b} $L {522 lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)]523 }524 lappend r2 $M525 }526 527 do_execsql_test 11.1.$tn.2 {528 SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr529 } $r2530 531 do_execsql_test 11.1.$tn.2 {532 SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr533 } $r2534}535set sqlite_fts3_enable_parentheses 0536 537#---------------------------------------------------------------------------538# Test the 'b' matchinfo flag539#540set sqlite_fts3_enable_parentheses 1541reset_db542db func mit mit543 544do_test 12.0 {545 set cols [list]546 for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }547 execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])"548} {}549 550do_execsql_test 12.1 {551 INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc');552 SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';553} [list [list [expr 1<<4] [expr 1<<(45-32)]]]554 555set sqlite_fts3_enable_parentheses 0556finish_test557