AryaWu/sqlite

1# 2007 June 212#3# The author disclaims copyright to this source code.  In place of4# a legal notice, here is a blessing:5#6#    May you do good and not evil.7#    May you find forgiveness for yourself and forgive others.8#    May you share freely, never taking more than you give.9#10#*************************************************************************11# This file implements regression tests for SQLite library. The focus 12# of this script is testing the pluggable tokeniser feature of the 13# FTS3 module.14#15# $Id: fts3atoken.test,v 1.1 2007/08/20 17:38:42 shess Exp $16#17 18set testdir [file dirname $argv0]19source $testdir/tester.tcl20 21# If SQLITE_ENABLE_FTS3 is defined, omit this file.22ifcapable !fts3 {23  finish_test24  return25}26 27set ::testprefix fts3atoken28 29proc escape_string {str} {30  set out ""31  foreach char [split $str ""] {32    scan $char %c i33    if {$i<=127} {34      append out $char35    } else {36      append out [format {\x%.4x} $i]37    }38  }39  set out40}41 42#--------------------------------------------------------------------------43# Test cases fts3atoken-1.* are the warm-body test for the SQL scalar44# function fts3_tokenizer(). The procedure is as follows:45#46#   1: Verify that there is no such fts3 tokenizer as 'blah'.47#48#   2: Query for the built-in tokenizer 'simple'. Insert a copy of the49#      retrieved value as tokenizer 'blah'.50#51#   3: Test that the value returned for tokenizer 'blah' is now the52#      same as that retrieved for 'simple'.53#54#   4: Test that it is now possible to create an fts3 table using 55#      tokenizer 'blah' (it was not possible in step 1).56#57#   5: Test that the table created to use tokenizer 'blah' is usable.58#59sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 160do_test fts3atoken-1.1 {61  catchsql {62    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);63  }64} {1 {unknown tokenizer: blah}}65do_test fts3atoken-1.2 {66  execsql {67    SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;68  }69} {0}70do_test fts3atoken-1.3 {71  execsql {72    SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');73  }74} {1}75do_test fts3atoken-1.4 {76  catchsql {77    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);78  }79} {0 {}}80do_test fts3atoken-1.5 {81  execsql {82    INSERT INTO t1(content) VALUES('There was movement at the station');83    INSERT INTO t1(content) VALUES('For the word has passed around');84    INSERT INTO t1(content) VALUES('That the colt from ol regret had got');85    SELECT content FROM t1 WHERE content MATCH 'movement'86  }87} {{There was movement at the station}}88 89unset -nocomplain simple blah2name simplename90set simplename "simple"91set blah2name "blah2"92set simple [db one {SELECT fts3_tokenizer('simple')}]93sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 094do_catchsql_test 1.6 {95  SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;96} {1 {fts3tokenize disabled}}97do_test fts3atoken-1.7 {98  execsql {99    SELECT fts3_tokenizer('blah2', $simple) IS NULL;100  }101} {1}102 103# With ENABLE_FTS3_TOKENIZER off, the fts3_tokenzer(1) function104# returns NULL unless the first parameter is a bound parameter.105# If the first parameter is a bound parameter, then fts3_tokenizer(1)106# returns the actual pointer value as a BLOB.107#108do_test fts3atoken-1.8 {109  execsql {110    SELECT fts3_tokenizer($blah2name) == fts3_tokenizer($simplename),111           typeof(fts3_tokenizer($blah2name)),112           typeof(fts3_tokenizer('blah2')),113           typeof(fts3_tokenizer($simplename)),114           typeof(fts3_tokenizer('simple'));115  }116} {1 blob null blob null}117 118# With ENABLE_FTS3_TOKENIZER on, fts3_tokenizer() always returns119# the BLOB pointer, regardless the parameter120#121sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1122do_test fts3atoken-1.9 {123  execsql {124    SELECT fts3_tokenizer('blah2') == fts3_tokenizer('simple'),125           typeof(fts3_tokenizer($blah2name)),126           typeof(fts3_tokenizer('blah2')),127           typeof(fts3_tokenizer($simplename)),128           typeof(fts3_tokenizer('simple'));129  }130} {1 blob blob blob blob}131 132# 2019-12-31:  The fts3_tokenizer() function can never be invoked from133# within a trigger or view.134#135do_catchsql_test fts3atoken-1.10 {136  CREATE VIEW v110(x) AS137      SELECT fts3_tokenizer('tok110', fts3_tokenizer('simple')) IS NULL;138} {0 {}}139do_catchsql_test fts3atoken-1.11 {140  SELECT * FROM v110;141} {1 {unsafe use of fts3_tokenizer()}}142do_catchsql_test fts3atoken-1.12 {143  CREATE TABLE t110(a,b);144  CREATE TRIGGER r110 AFTER INSERT ON t110 BEGIN145      SELECT fts3_tokenizer('tok110', fts3_tokenizer('simple')) IS NULL;146  END;147} {0 {}}148do_catchsql_test fts3atoken-1.13 {149  INSERT INTO t110(a,b) VALUES(1,2);150} {1 {unsafe use of fts3_tokenizer()}}151do_catchsql_test fts3atoken-1.14 {152  SELECT * FROM t110;153} {0 {}}154 155#--------------------------------------------------------------------------156# Test cases fts3atoken-2.* test error cases in the scalar function based157# API for getting and setting tokenizers.158#159do_test fts3atoken-2.1 {160  catchsql {161    SELECT fts3_tokenizer('nosuchtokenizer');162  }163} {1 {unknown tokenizer: nosuchtokenizer}}164 165#--------------------------------------------------------------------------166# Test cases fts3atoken-3.* test the three built-in tokenizers with a167# simple input string via the built-in test function. This is as much168# to test the test function as the tokenizer implementations.169#170sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1171do_test fts3atoken-3.1 {172  execsql {173    SELECT fts3_tokenizer_test('simple', 'I don''t see how');174  }175} {{0 i I 1 don don 2 t t 3 see see 4 how how}}176do_test fts3atoken-3.2 {177  execsql {178    SELECT fts3_tokenizer_test('porter', 'I don''t see how');179  }180} {{0 i I 1 don don 2 t t 3 see see 4 how how}}181ifcapable icu {182  do_test fts3atoken-3.3 {183    execsql {184      SELECT fts3_tokenizer_test('icu', 'I don''t see how');185    }186  } {{0 i I 1 don't don't 2 see see 3 how how}}187}188 189#--------------------------------------------------------------------------190# Test cases fts3atoken-4.* test the ICU tokenizer. In practice, this191# tokenizer only has two modes - "thai" and "everybody else". Some other192# Asian languages (Lao, Khmer etc.) require the same special treatment as 193# Thai, but ICU doesn't support them yet.194#195ifcapable icu {196 197  proc do_icu_test {name locale input output} {198    set ::out [db eval { SELECT fts3_tokenizer_test('icu', $locale, $input) }]199    do_test $name {200      lindex $::out 0201    } $output202  }203  204  do_icu_test fts3atoken-4.1 en_US  {}   {}205  do_icu_test fts3atoken-4.2 en_US {Test cases fts3} [list \206    0 test Test 1 cases cases 2 fts3 fts3207  ]208 209  # The following test shows that ICU is smart enough to recognise210  # Thai chararacters, even when the locale is set to English/United 211  # States.212  #213  set input "\u0e2d\u0e30\u0e44\u0e23\u0e19\u0e30\u0e04\u0e23\u0e31\u0e1a"214  set output    "0 \u0e2d\u0e30\u0e44\u0e23 \u0e2d\u0e30\u0e44\u0e23 "215  append output "1 \u0e19\u0e30 \u0e19\u0e30 "216  append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a"217 218  do_icu_test fts3atoken-4.3 th_TH  $input $output219  do_icu_test fts3atoken-4.4 en_US  $input $output220 221  # ICU handles an unknown locale by falling back to the default.222  # So this is not an error.223  do_icu_test fts3atoken-4.5 MiddleOfTheOcean  $input $output224 225  set    longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire"226  append longtoken "AReallocInTheIcuTokenizerCode"227 228  set    input "short tokens then "229  append input $longtoken230  set    output "0 short short "231  append output "1 tokens tokens "232  append output "2 then then "233  append output "3 [string tolower $longtoken] $longtoken"234 235  do_icu_test fts3atoken-4.6 MiddleOfTheOcean  $input $output236  do_icu_test fts3atoken-4.7 th_TH  $input $output237  do_icu_test fts3atoken-4.8 en_US  $input $output238 239  do_execsql_test 5.1 {240    CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);241    insert into x1 (name) values (NULL);242    insert into x1 (name) values (NULL);243    delete from x1;244  }245 246  proc cp_to_str {codepoint_list} {247    set fmt [string repeat %c [llength $codepoint_list]]248    eval [list format $fmt] $codepoint_list249  }250 251  do_test 5.2 {252    set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}]253    execsql { INSERT INTO x1 VALUES($str) }254  } {}255}256 257do_test fts3atoken-internal {258  execsql { SELECT fts3_tokenizer_internal_test() }259} {ok}260 261#-------------------------------------------------------------------------262# Test empty tokenizer names.263#264do_catchsql_test 6.1.1 {265  CREATE VIRTUAL TABLE t3 USING fts4(tokenize="");266} {1 {unknown tokenizer: }}267do_catchsql_test 6.1.2 {268  CREATE VIRTUAL TABLE t3 USING fts4(tokenize=);269} {1 {unknown tokenizer: }}270do_catchsql_test 6.1.3 {271  CREATE VIRTUAL TABLE t3 USING fts4(tokenize="   ");272} {1 {unknown tokenizer:    }}273 274do_catchsql_test 6.2.1 {275  SELECT fts3_tokenizer(NULL);276} {1 {unknown tokenizer: }}277 278sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1279do_catchsql_test 6.2.2 {280  SELECT fts3_tokenizer(NULL, X'1234567812345678');281} {1 {argument type mismatch}}282do_catchsql_test 6.2.3 {283  SELECT fts3_tokenizer(NULL, X'12345678');284} {1 {argument type mismatch}}285 286 287finish_test288