AryaWu/sqlite
0
1 2/*3** This, the "fts5.c" source file, is a composite file that is itself4** assembled from the following files:5**6** fts5.h7** fts5Int.h8** fts5parse.h <--- Generated from fts5parse.y by Lemon9** fts5parse.c <--- Generated from fts5parse.y by Lemon10** fts5_aux.c11** fts5_buffer.c12** fts5_config.c13** fts5_expr.c14** fts5_hash.c15** fts5_index.c16** fts5_main.c17** fts5_storage.c18** fts5_tokenize.c19** fts5_unicode2.c20** fts5_varint.c21** fts5_vocab.c22*/23#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) 24 25#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 26# define NDEBUG 127#endif28#if defined(NDEBUG) && defined(SQLITE_DEBUG)29# undef NDEBUG30#endif31 32#ifdef HAVE_STDINT_H33#include <stdint.h>34#endif35#ifdef HAVE_INTTYPES_H36#include <inttypes.h>37#endif38#line 1 "fts5.h"39/*40** 2014 May 3141**42** The author disclaims copyright to this source code. In place of43** a legal notice, here is a blessing:44**45** May you do good and not evil.46** May you find forgiveness for yourself and forgive others.47** May you share freely, never taking more than you give.48**49******************************************************************************50**51** Interfaces to extend FTS5. Using the interfaces defined in this file, 52** FTS5 may be extended with:53**54** * custom tokenizers, and55** * custom auxiliary functions.56*/57 58 59#ifndef _FTS5_H60#define _FTS5_H61 62#include "sqlite3.h"63 64#ifdef __cplusplus65extern "C" {66#endif67 68/*************************************************************************69** CUSTOM AUXILIARY FUNCTIONS70**71** Virtual table implementations may overload SQL functions by implementing72** the sqlite3_module.xFindFunction() method.73*/74 75typedef struct Fts5ExtensionApi Fts5ExtensionApi;76typedef struct Fts5Context Fts5Context;77typedef struct Fts5PhraseIter Fts5PhraseIter;78 79typedef void (*fts5_extension_function)(80 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */81 Fts5Context *pFts, /* First arg to pass to pApi functions */82 sqlite3_context *pCtx, /* Context for returning result/error */83 int nVal, /* Number of values in apVal[] array */84 sqlite3_value **apVal /* Array of trailing arguments */85);86 87struct Fts5PhraseIter {88 const unsigned char *a;89 const unsigned char *b;90};91 92/*93** EXTENSION API FUNCTIONS94**95** xUserData(pFts):96** Return a copy of the pUserData pointer passed to the xCreateFunction()97** API when the extension function was registered.98**99** xColumnTotalSize(pFts, iCol, pnToken):100** If parameter iCol is less than zero, set output variable *pnToken101** to the total number of tokens in the FTS5 table. Or, if iCol is102** non-negative but less than the number of columns in the table, return103** the total number of tokens in column iCol, considering all rows in 104** the FTS5 table.105**106** If parameter iCol is greater than or equal to the number of columns107** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.108** an OOM condition or IO error), an appropriate SQLite error code is 109** returned.110**111** xColumnCount(pFts):112** Return the number of columns in the table.113**114** xColumnSize(pFts, iCol, pnToken):115** If parameter iCol is less than zero, set output variable *pnToken116** to the total number of tokens in the current row. Or, if iCol is117** non-negative but less than the number of columns in the table, set118** *pnToken to the number of tokens in column iCol of the current row.119**120** If parameter iCol is greater than or equal to the number of columns121** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.122** an OOM condition or IO error), an appropriate SQLite error code is 123** returned.124**125** This function may be quite inefficient if used with an FTS5 table126** created with the "columnsize=0" option.127**128** xColumnText:129** If parameter iCol is less than zero, or greater than or equal to the130** number of columns in the table, SQLITE_RANGE is returned. 131**132** Otherwise, this function attempts to retrieve the text of column iCol of133** the current document. If successful, (*pz) is set to point to a buffer134** containing the text in utf-8 encoding, (*pn) is set to the size in bytes135** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,136** if an error occurs, an SQLite error code is returned and the final values137** of (*pz) and (*pn) are undefined.138**139** xPhraseCount:140** Returns the number of phrases in the current query expression.141**142** xPhraseSize:143** If parameter iCol is less than zero, or greater than or equal to the144** number of phrases in the current query, as returned by xPhraseCount, 145** 0 is returned. Otherwise, this function returns the number of tokens in146** phrase iPhrase of the query. Phrases are numbered starting from zero.147**148** xInstCount:149** Set *pnInst to the total number of occurrences of all phrases within150** the query within the current row. Return SQLITE_OK if successful, or151** an error code (i.e. SQLITE_NOMEM) if an error occurs.152**153** This API can be quite slow if used with an FTS5 table created with the154** "detail=none" or "detail=column" option. If the FTS5 table is created 155** with either "detail=none" or "detail=column" and "content=" option 156** (i.e. if it is a contentless table), then this API always returns 0.157**158** xInst:159** Query for the details of phrase match iIdx within the current row.160** Phrase matches are numbered starting from zero, so the iIdx argument161** should be greater than or equal to zero and smaller than the value162** output by xInstCount(). If iIdx is less than zero or greater than163** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.164**165** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol166** to the column in which it occurs and *piOff the token offset of the167** first token of the phrase. SQLITE_OK is returned if successful, or an168** error code (i.e. SQLITE_NOMEM) if an error occurs.169**170** This API can be quite slow if used with an FTS5 table created with the171** "detail=none" or "detail=column" option. 172**173** xRowid:174** Returns the rowid of the current row.175**176** xTokenize:177** Tokenize text using the tokenizer belonging to the FTS5 table.178**179** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):180** This API function is used to query the FTS table for phrase iPhrase181** of the current query. Specifically, a query equivalent to:182**183** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid184**185** with $p set to a phrase equivalent to the phrase iPhrase of the186** current query is executed. Any column filter that applies to187** phrase iPhrase of the current query is included in $p. For each 188** row visited, the callback function passed as the fourth argument 189** is invoked. The context and API objects passed to the callback 190** function may be used to access the properties of each matched row.191** Invoking Api.xUserData() returns a copy of the pointer passed as 192** the third argument to pUserData.193**194** If parameter iPhrase is less than zero, or greater than or equal to195** the number of phrases in the query, as returned by xPhraseCount(),196** this function returns SQLITE_RANGE.197**198** If the callback function returns any value other than SQLITE_OK, the199** query is abandoned and the xQueryPhrase function returns immediately.200** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.201** Otherwise, the error code is propagated upwards.202**203** If the query runs to completion without incident, SQLITE_OK is returned.204** Or, if some error occurs before the query completes or is aborted by205** the callback, an SQLite error code is returned.206**207**208** xSetAuxdata(pFts5, pAux, xDelete)209**210** Save the pointer passed as the second argument as the extension function's 211** "auxiliary data". The pointer may then be retrieved by the current or any212** future invocation of the same fts5 extension function made as part of213** the same MATCH query using the xGetAuxdata() API.214**215** Each extension function is allocated a single auxiliary data slot for216** each FTS query (MATCH expression). If the extension function is invoked 217** more than once for a single FTS query, then all invocations share a 218** single auxiliary data context.219**220** If there is already an auxiliary data pointer when this function is221** invoked, then it is replaced by the new pointer. If an xDelete callback222** was specified along with the original pointer, it is invoked at this223** point.224**225** The xDelete callback, if one is specified, is also invoked on the226** auxiliary data pointer after the FTS5 query has finished.227**228** If an error (e.g. an OOM condition) occurs within this function,229** the auxiliary data is set to NULL and an error code returned. If the230** xDelete parameter was not NULL, it is invoked on the auxiliary data231** pointer before returning.232**233**234** xGetAuxdata(pFts5, bClear)235**236** Returns the current auxiliary data pointer for the fts5 extension 237** function. See the xSetAuxdata() method for details.238**239** If the bClear argument is non-zero, then the auxiliary data is cleared240** (set to NULL) before this function returns. In this case the xDelete,241** if any, is not invoked.242**243**244** xRowCount(pFts5, pnRow)245**246** This function is used to retrieve the total number of rows in the table.247** In other words, the same value that would be returned by:248**249** SELECT count(*) FROM ftstable;250**251** xPhraseFirst()252** This function is used, along with type Fts5PhraseIter and the xPhraseNext253** method, to iterate through all instances of a single query phrase within254** the current row. This is the same information as is accessible via the255** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient256** to use, this API may be faster under some circumstances. To iterate 257** through instances of phrase iPhrase, use the following code:258**259** Fts5PhraseIter iter;260** int iCol, iOff;261** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);262** iCol>=0;263** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)264** ){265** // An instance of phrase iPhrase at offset iOff of column iCol266** }267**268** The Fts5PhraseIter structure is defined above. Applications should not269** modify this structure directly - it should only be used as shown above270** with the xPhraseFirst() and xPhraseNext() API methods (and by271** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).272**273** This API can be quite slow if used with an FTS5 table created with the274** "detail=none" or "detail=column" option. If the FTS5 table is created 275** with either "detail=none" or "detail=column" and "content=" option 276** (i.e. if it is a contentless table), then this API always iterates277** through an empty set (all calls to xPhraseFirst() set iCol to -1).278**279** In all cases, matches are visited in (column ASC, offset ASC) order.280** i.e. all those in column 0, sorted by offset, followed by those in 281** column 1, etc.282**283** xPhraseNext()284** See xPhraseFirst above.285**286** xPhraseFirstColumn()287** This function and xPhraseNextColumn() are similar to the xPhraseFirst()288** and xPhraseNext() APIs described above. The difference is that instead289** of iterating through all instances of a phrase in the current row, these290** APIs are used to iterate through the set of columns in the current row291** that contain one or more instances of a specified phrase. For example:292**293** Fts5PhraseIter iter;294** int iCol;295** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);296** iCol>=0;297** pApi->xPhraseNextColumn(pFts, &iter, &iCol)298** ){299** // Column iCol contains at least one instance of phrase iPhrase300** }301**302** This API can be quite slow if used with an FTS5 table created with the303** "detail=none" option. If the FTS5 table is created with either 304** "detail=none" "content=" option (i.e. if it is a contentless table), 305** then this API always iterates through an empty set (all calls to 306** xPhraseFirstColumn() set iCol to -1).307**308** The information accessed using this API and its companion309** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext310** (or xInst/xInstCount). The chief advantage of this API is that it is311** significantly more efficient than those alternatives when used with312** "detail=column" tables. 313**314** xPhraseNextColumn()315** See xPhraseFirstColumn above.316**317** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)318** This is used to access token iToken of phrase iPhrase of the current319** query. Before returning, output parameter *ppToken is set to point320** to a buffer containing the requested token, and *pnToken to the321** size of this buffer in bytes.322**323** If iPhrase or iToken are less than zero, or if iPhrase is greater than324** or equal to the number of phrases in the query as reported by 325** xPhraseCount(), or if iToken is equal to or greater than the number of326** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken327 are both zeroed.328**329** The output text is not a copy of the query text that specified the330** token. It is the output of the tokenizer module. For tokendata=1331** tables, this includes any embedded 0x00 and trailing data.332**333** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)334** This is used to access token iToken of phrase hit iIdx within the335** current row. If iIdx is less than zero or greater than or equal to the336** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise,337** output variable (*ppToken) is set to point to a buffer containing the338** matching document token, and (*pnToken) to the size of that buffer in 339** bytes. 340**341** The output text is not a copy of the document text that was tokenized.342** It is the output of the tokenizer module. For tokendata=1 tables, this 343** includes any embedded 0x00 and trailing data.344**345** This API may be slow in some cases if the token identified by parameters 346** iIdx and iToken matched a prefix token in the query. In most cases, the347** first call to this API for each prefix token in the query is forced348** to scan the portion of the full-text index that matches the prefix349** token to collect the extra data required by this API. If the prefix350** token matches a large number of token instances in the document set,351** this may be a performance problem. 352**353** If the user knows in advance that a query may use this API for a354** prefix token, FTS5 may be configured to collect all required data as part355** of the initial querying of the full-text index, avoiding the second scan356** entirely. This also causes prefix queries that do not use this API to 357** run more slowly and use more memory. FTS5 may be configured in this way358** either on a per-table basis using the [FTS5 insttoken | 'insttoken'] 359** option, or on a per-query basis using the 360** [fts5_insttoken | fts5_insttoken()] user function.361**362** This API can be quite slow if used with an FTS5 table created with the363** "detail=none" or "detail=column" option. 364**365** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)366** If parameter iCol is less than zero, or greater than or equal to the367** number of columns in the table, SQLITE_RANGE is returned.368**369** Otherwise, this function attempts to retrieve the locale associated370** with column iCol of the current row. Usually, there is no associated371** locale, and output parameters (*pzLocale) and (*pnLocale) are set372** to NULL and 0, respectively. However, if the fts5_locale() function373** was used to associate a locale with the value when it was inserted374** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated375** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) 376** is set to the size in bytes of the buffer, not including the 377** nul-terminator.378**379** If successful, SQLITE_OK is returned. Or, if an error occurs, an380** SQLite error code is returned. The final value of the output parameters381** is undefined in this case.382**383** xTokenize_v2:384** Tokenize text using the tokenizer belonging to the FTS5 table. This385** API is the same as the xTokenize() API, except that it allows a tokenizer386** locale to be specified.387*/388struct Fts5ExtensionApi {389 int iVersion; /* Currently always set to 4 */390 391 void *(*xUserData)(Fts5Context*);392 393 int (*xColumnCount)(Fts5Context*);394 int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);395 int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);396 397 int (*xTokenize)(Fts5Context*, 398 const char *pText, int nText, /* Text to tokenize */399 void *pCtx, /* Context passed to xToken() */400 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */401 );402 403 int (*xPhraseCount)(Fts5Context*);404 int (*xPhraseSize)(Fts5Context*, int iPhrase);405 406 int (*xInstCount)(Fts5Context*, int *pnInst);407 int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);408 409 sqlite3_int64 (*xRowid)(Fts5Context*);410 int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);411 int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);412 413 int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,414 int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)415 );416 int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));417 void *(*xGetAuxdata)(Fts5Context*, int bClear);418 419 int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);420 void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);421 422 int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);423 void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);424 425 /* Below this point are iVersion>=3 only */426 int (*xQueryToken)(Fts5Context*, 427 int iPhrase, int iToken, 428 const char **ppToken, int *pnToken429 );430 int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);431 432 /* Below this point are iVersion>=4 only */433 int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);434 int (*xTokenize_v2)(Fts5Context*,435 const char *pText, int nText, /* Text to tokenize */436 const char *pLocale, int nLocale, /* Locale to pass to tokenizer */437 void *pCtx, /* Context passed to xToken() */438 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */439 );440};441 442/* 443** CUSTOM AUXILIARY FUNCTIONS444*************************************************************************/445 446/*************************************************************************447** CUSTOM TOKENIZERS448**449** Applications may also register custom tokenizer types. A tokenizer 450** is registered by providing fts5 with a populated instance of the 451** following structure. All structure methods must be defined, setting452** any member of the fts5_tokenizer struct to NULL leads to undefined453** behaviour. The structure methods are expected to function as follows:454**455** xCreate:456** This function is used to allocate and initialize a tokenizer instance.457** A tokenizer instance is required to actually tokenize text.458**459** The first argument passed to this function is a copy of the (void*)460** pointer provided by the application when the fts5_tokenizer_v2 object461** was registered with FTS5 (the third argument to xCreateTokenizer()). 462** The second and third arguments are an array of nul-terminated strings463** containing the tokenizer arguments, if any, specified following the464** tokenizer name as part of the CREATE VIRTUAL TABLE statement used465** to create the FTS5 table.466**467** The final argument is an output variable. If successful, (*ppOut) 468** should be set to point to the new tokenizer handle and SQLITE_OK469** returned. If an error occurs, some value other than SQLITE_OK should470** be returned. In this case, fts5 assumes that the final value of *ppOut 471** is undefined.472**473** xDelete:474** This function is invoked to delete a tokenizer handle previously475** allocated using xCreate(). Fts5 guarantees that this function will476** be invoked exactly once for each successful call to xCreate().477**478** xTokenize:479** This function is expected to tokenize the nText byte string indicated 480** by argument pText. pText may or may not be nul-terminated. The first481** argument passed to this function is a pointer to an Fts5Tokenizer object482** returned by an earlier call to xCreate().483**484** The third argument indicates the reason that FTS5 is requesting485** tokenization of the supplied text. This is always one of the following486** four values:487**488** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into489** or removed from the FTS table. The tokenizer is being invoked to490** determine the set of tokens to add to (or delete from) the491** FTS index.492**493** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed 494** against the FTS index. The tokenizer is being called to tokenize 495** a bareword or quoted string specified as part of the query.496**497** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as498** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is499** followed by a "*" character, indicating that the last token500** returned by the tokenizer will be treated as a token prefix.501**502** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to 503** satisfy an fts5_api.xTokenize() request made by an auxiliary504** function. Or an fts5_api.xColumnSize() request made by the same505** on a columnsize=0 database. 506** </ul>507**508** The sixth and seventh arguments passed to xTokenize() - pLocale and509** nLocale - are a pointer to a buffer containing the locale to use for510** tokenization (e.g. "en_US") and its size in bytes, respectively. The511** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in512** which case nLocale is always 0) to indicate that the tokenizer should513** use its default locale.514**515** For each token in the input string, the supplied callback xToken() must516** be invoked. The first argument to it should be a copy of the pointer517** passed as the second argument to xTokenize(). The third and fourth518** arguments are a pointer to a buffer containing the token text, and the519** size of the token in bytes. The 4th and 5th arguments are the byte offsets520** of the first byte of and first byte immediately following the text from521** which the token is derived within the input.522**523** The second argument passed to the xToken() callback ("tflags") should524** normally be set to 0. The exception is if the tokenizer supports 525** synonyms. In this case see the discussion below for details.526**527** FTS5 assumes the xToken() callback is invoked for each token in the 528** order that they occur within the input text.529**530** If an xToken() callback returns any value other than SQLITE_OK, then531** the tokenization should be abandoned and the xTokenize() method should532** immediately return a copy of the xToken() return value. Or, if the533** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,534** if an error occurs with the xTokenize() implementation itself, it535** may abandon the tokenization and return any error code other than536** SQLITE_OK or SQLITE_DONE.537**538** If the tokenizer is registered using an fts5_tokenizer_v2 object,539** then the xTokenize() method has two additional arguments - pLocale540** and nLocale. These specify the locale that the tokenizer should use541** for the current request. If pLocale and nLocale are both 0, then the542** tokenizer should use its default locale. Otherwise, pLocale points to543** an nLocale byte buffer containing the name of the locale to use as utf-8 544** text. pLocale is not nul-terminated.545**546** FTS5_TOKENIZER547**548** There is also an fts5_tokenizer object. This is an older, deprecated,549** version of fts5_tokenizer_v2. It is similar except that:550**551** <ul>552** <li> There is no "iVersion" field, and553** <li> The xTokenize() method does not take a locale argument.554** </ul>555**556** Legacy fts5_tokenizer tokenizers must be registered using the557** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().558**559** Tokenizer implementations registered using either API may be retrieved560** using both xFindTokenizer() and xFindTokenizer_v2().561**562** SYNONYM SUPPORT563**564** Custom tokenizers may also support synonyms. Consider a case in which a565** user wishes to query for a phrase such as "first place". Using the 566** built-in tokenizers, the FTS5 query 'first + place' will match instances567** of "first place" within the document set, but not alternative forms568** such as "1st place". In some applications, it would be better to match569** all instances of "first place" or "1st place" regardless of which form570** the user specified in the MATCH query text.571**572** There are several ways to approach this in FTS5:573**574** <ol><li> By mapping all synonyms to a single token. In this case, using575** the above example, this means that the tokenizer returns the576** same token for inputs "first" and "1st". Say that token is in577** fact "first", so that when the user inserts the document "I won578** 1st place" entries are added to the index for tokens "i", "won",579** "first" and "place". If the user then queries for '1st + place',580** the tokenizer substitutes "first" for "1st" and the query works581** as expected.582**583** <li> By querying the index for all synonyms of each query term584** separately. In this case, when tokenizing query text, the585** tokenizer may provide multiple synonyms for a single term 586** within the document. FTS5 then queries the index for each 587** synonym individually. For example, faced with the query:588**589** <codeblock>590** ... MATCH 'first place'</codeblock>591**592** the tokenizer offers both "1st" and "first" as synonyms for the593** first token in the MATCH query and FTS5 effectively runs a query 594** similar to:595**596** <codeblock>597** ... MATCH '(first OR 1st) place'</codeblock>598**599** except that, for the purposes of auxiliary functions, the query600** still appears to contain just two phrases - "(first OR 1st)" 601** being treated as a single phrase.602**603** <li> By adding multiple synonyms for a single term to the FTS index.604** Using this method, when tokenizing document text, the tokenizer605** provides multiple synonyms for each token. So that when a 606** document such as "I won first place" is tokenized, entries are607** added to the FTS index for "i", "won", "first", "1st" and608** "place".609**610** This way, even if the tokenizer does not provide synonyms611** when tokenizing query text (it should not - to do so would be612** inefficient), it doesn't matter if the user queries for 613** 'first + place' or '1st + place', as there are entries in the614** FTS index corresponding to both forms of the first token.615** </ol>616**617** Whether it is parsing document or query text, any call to xToken that618** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit619** is considered to supply a synonym for the previous token. For example,620** when parsing the document "I won first place", a tokenizer that supports621** synonyms would call xToken() 5 times, as follows:622**623** <codeblock>624** xToken(pCtx, 0, "i", 1, 0, 1);625** xToken(pCtx, 0, "won", 3, 2, 5);626** xToken(pCtx, 0, "first", 5, 6, 11);627** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);628** xToken(pCtx, 0, "place", 5, 12, 17);629**</codeblock>630**631** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time632** xToken() is called. Multiple synonyms may be specified for a single token633** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. 634** There is no limit to the number of synonyms that may be provided for a635** single token.636**637** In many cases, method (1) above is the best approach. It does not add 638** extra data to the FTS index or require FTS5 to query for multiple terms,639** so it is efficient in terms of disk space and query speed. However, it640** does not support prefix queries very well. If, as suggested above, the641** token "first" is substituted for "1st" by the tokenizer, then the query:642**643** <codeblock>644** ... MATCH '1s*'</codeblock>645**646** will not match documents that contain the token "1st" (as the tokenizer647** will probably not map "1s" to any prefix of "first").648**649** For full prefix support, method (3) may be preferred. In this case, 650** because the index contains entries for both "first" and "1st", prefix651** queries such as 'fi*' or '1s*' will match correctly. However, because652** extra entries are added to the FTS index, this method uses more space653** within the database.654**655** Method (2) offers a midpoint between (1) and (3). Using this method,656** a query such as '1s*' will match documents that contain the literal 657** token "1st", but not "first" (assuming the tokenizer is not able to658** provide synonyms for prefixes). However, a non-prefix query like '1st'659** will match against "1st" and "first". This method does not require660** extra disk space, as no extra entries are added to the FTS index. 661** On the other hand, it may require more CPU cycles to run MATCH queries,662** as separate queries of the FTS index are required for each synonym.663**664** When using methods (2) or (3), it is important that the tokenizer only665** provide synonyms when tokenizing document text (method (3)) or query666** text (method (2)), not both. Doing so will not cause any errors, but is667** inefficient.668*/669typedef struct Fts5Tokenizer Fts5Tokenizer;670typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;671struct fts5_tokenizer_v2 {672 int iVersion; /* Currently always 2 */673 674 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);675 void (*xDelete)(Fts5Tokenizer*);676 int (*xTokenize)(Fts5Tokenizer*, 677 void *pCtx,678 int flags, /* Mask of FTS5_TOKENIZE_* flags */679 const char *pText, int nText, 680 const char *pLocale, int nLocale,681 int (*xToken)(682 void *pCtx, /* Copy of 2nd argument to xTokenize() */683 int tflags, /* Mask of FTS5_TOKEN_* flags */684 const char *pToken, /* Pointer to buffer containing token */685 int nToken, /* Size of token in bytes */686 int iStart, /* Byte offset of token within input text */687 int iEnd /* Byte offset of end of token within input text */688 )689 );690};691 692/*693** New code should use the fts5_tokenizer_v2 type to define tokenizer694** implementations. The following type is included for legacy applications695** that still use it.696*/697typedef struct fts5_tokenizer fts5_tokenizer;698struct fts5_tokenizer {699 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);700 void (*xDelete)(Fts5Tokenizer*);701 int (*xTokenize)(Fts5Tokenizer*, 702 void *pCtx,703 int flags, /* Mask of FTS5_TOKENIZE_* flags */704 const char *pText, int nText, 705 int (*xToken)(706 void *pCtx, /* Copy of 2nd argument to xTokenize() */707 int tflags, /* Mask of FTS5_TOKEN_* flags */708 const char *pToken, /* Pointer to buffer containing token */709 int nToken, /* Size of token in bytes */710 int iStart, /* Byte offset of token within input text */711 int iEnd /* Byte offset of end of token within input text */712 )713 );714};715 716 717/* Flags that may be passed as the third argument to xTokenize() */718#define FTS5_TOKENIZE_QUERY 0x0001719#define FTS5_TOKENIZE_PREFIX 0x0002720#define FTS5_TOKENIZE_DOCUMENT 0x0004721#define FTS5_TOKENIZE_AUX 0x0008722 723/* Flags that may be passed by the tokenizer implementation back to FTS5724** as the third argument to the supplied xToken callback. */725#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */726 727/*728** END OF CUSTOM TOKENIZERS729*************************************************************************/730 731/*************************************************************************732** FTS5 EXTENSION REGISTRATION API733*/734typedef struct fts5_api fts5_api;735struct fts5_api {736 int iVersion; /* Currently always set to 3 */737 738 /* Create a new tokenizer */739 int (*xCreateTokenizer)(740 fts5_api *pApi,741 const char *zName,742 void *pUserData,743 fts5_tokenizer *pTokenizer,744 void (*xDestroy)(void*)745 );746 747 /* Find an existing tokenizer */748 int (*xFindTokenizer)(749 fts5_api *pApi,750 const char *zName,751 void **ppUserData,752 fts5_tokenizer *pTokenizer753 );754 755 /* Create a new auxiliary function */756 int (*xCreateFunction)(757 fts5_api *pApi,758 const char *zName,759 void *pUserData,760 fts5_extension_function xFunction,761 void (*xDestroy)(void*)762 );763 764 /* APIs below this point are only available if iVersion>=3 */765 766 /* Create a new tokenizer */767 int (*xCreateTokenizer_v2)(768 fts5_api *pApi,769 const char *zName,770 void *pUserData,771 fts5_tokenizer_v2 *pTokenizer,772 void (*xDestroy)(void*)773 );774 775 /* Find an existing tokenizer */776 int (*xFindTokenizer_v2)(777 fts5_api *pApi,778 const char *zName,779 void **ppUserData,780 fts5_tokenizer_v2 **ppTokenizer781 );782};783 784/*785** END OF REGISTRATION API786*************************************************************************/787 788#ifdef __cplusplus789} /* end of the 'extern "C"' block */790#endif791 792#endif /* _FTS5_H */793 794#line 1 "fts5Int.h"795/*796** 2014 May 31797**798** The author disclaims copyright to this source code. In place of799** a legal notice, here is a blessing:800**801** May you do good and not evil.802** May you find forgiveness for yourself and forgive others.803** May you share freely, never taking more than you give.804**805******************************************************************************806**807*/808#ifndef _FTS5INT_H809#define _FTS5INT_H810 811/* #include "fts5.h" */812#include "sqlite3ext.h"813SQLITE_EXTENSION_INIT1814 815#include <string.h>816#include <assert.h>817#include <stddef.h>818 819#ifndef SQLITE_AMALGAMATION820 821typedef unsigned char u8;822typedef unsigned int u32;823typedef unsigned short u16;824typedef short i16;825typedef sqlite3_int64 i64;826typedef sqlite3_uint64 u64;827 828#ifndef ArraySize829# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))830#endif831 832#define testcase(x)833 834#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)835# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1836#endif837#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)838# define ALWAYS(X) (1)839# define NEVER(X) (0)840#elif !defined(NDEBUG)841# define ALWAYS(X) ((X)?1:(assert(0),0))842# define NEVER(X) ((X)?(assert(0),1):0)843#else844# define ALWAYS(X) (X)845# define NEVER(X) (X)846#endif847 848#define MIN(x,y) (((x) < (y)) ? (x) : (y))849#define MAX(x,y) (((x) > (y)) ? (x) : (y))850 851/*852** Constants for the largest and smallest possible 64-bit signed integers.853*/854# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))855# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)856 857/*858** This macro is used in a single assert() within fts5 to check that an859** allocation is aligned to an 8-byte boundary. But it is a complicated860** macro to get right for multiple platforms without generating warnings.861** So instead of reproducing the entire definition from sqliteInt.h, we862** just do without this assert() for the rare non-amalgamation builds.863*/864#define EIGHT_BYTE_ALIGNMENT(x) 1865 866/*867** Macros needed to provide flexible arrays in a portable way868*/869#ifndef offsetof870# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0))871#endif872#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)873# define FLEXARRAY874#else875# define FLEXARRAY 1876#endif877 878#endif879 880/* Truncate very long tokens to this many bytes. Hard limit is 881** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset882** field that occurs at the start of each leaf page (see fts5_index.c). */883#define FTS5_MAX_TOKEN_SIZE 32768884 885/*886** Maximum number of prefix indexes on single FTS5 table. This must be887** less than 32. If it is set to anything large than that, an #error888** directive in fts5_index.c will cause the build to fail.889*/890#define FTS5_MAX_PREFIX_INDEXES 31891 892/*893** Maximum segments permitted in a single index 894*/895#define FTS5_MAX_SEGMENT 2000896 897#define FTS5_DEFAULT_NEARDIST 10898#define FTS5_DEFAULT_RANK "bm25"899 900/* Name of rank and rowid columns */901#define FTS5_RANK_NAME "rank"902#define FTS5_ROWID_NAME "rowid"903 904#ifdef SQLITE_DEBUG905# define FTS5_CORRUPT sqlite3Fts5Corrupt()906static int sqlite3Fts5Corrupt(void);907#else908# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB909#endif910 911/*912** The assert_nc() macro is similar to the assert() macro, except that it913** is used for assert() conditions that are true only if it can be 914** guranteed that the database is not corrupt.915*/916#ifdef SQLITE_DEBUG917extern int sqlite3_fts5_may_be_corrupt;918# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))919#else920# define assert_nc(x) assert(x)921#endif922 923/*924** A version of memcmp() that does not cause asan errors if one of the pointer925** parameters is NULL and the number of bytes to compare is zero.926*/927#define fts5Memcmp(s1, s2, n) ((n)<=0 ? 0 : memcmp((s1), (s2), (n)))928 929/* Mark a function parameter as unused, to suppress nuisance compiler930** warnings. */931#ifndef UNUSED_PARAM932# define UNUSED_PARAM(X) (void)(X)933#endif934 935#ifndef UNUSED_PARAM2936# define UNUSED_PARAM2(X, Y) (void)(X), (void)(Y)937#endif938 939typedef struct Fts5Global Fts5Global;940typedef struct Fts5Colset Fts5Colset;941 942/* If a NEAR() clump or phrase may only match a specific set of columns, 943** then an object of the following type is used to record the set of columns.944** Each entry in the aiCol[] array is a column that may be matched.945**946** This object is used by fts5_expr.c and fts5_index.c.947*/948struct Fts5Colset {949 int nCol;950 int aiCol[FLEXARRAY];951};952 953/* Size (int bytes) of a complete Fts5Colset object with N columns. */954#define SZ_FTS5COLSET(N) (sizeof(i64)*((N+2)/2))955 956/**************************************************************************957** Interface to code in fts5_config.c. fts5_config.c contains contains code958** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.959*/960 961typedef struct Fts5Config Fts5Config;962typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;963 964struct Fts5TokenizerConfig {965 Fts5Tokenizer *pTok;966 fts5_tokenizer_v2 *pApi2;967 fts5_tokenizer *pApi1;968 const char **azArg;969 int nArg;970 int ePattern; /* FTS_PATTERN_XXX constant */971 const char *pLocale; /* Current locale to use */972 int nLocale; /* Size of pLocale in bytes */973};974 975/*976** An instance of the following structure encodes all information that can977** be gleaned from the CREATE VIRTUAL TABLE statement.978**979** And all information loaded from the %_config table.980**981** nAutomerge:982** The minimum number of segments that an auto-merge operation should983** attempt to merge together. A value of 1 sets the object to use the 984** compile time default. Zero disables auto-merge altogether.985**986** bContentlessDelete:987** True if the contentless_delete option was present in the CREATE 988** VIRTUAL TABLE statement.989**990** zContent:991**992** zContentRowid:993** The value of the content_rowid= option, if one was specified. Or 994** the string "rowid" otherwise. This text is not quoted - if it is995** used as part of an SQL statement it needs to be quoted appropriately.996**997** zContentExprlist:998**999** pzErrmsg:1000** This exists in order to allow the fts5_index.c module to return a 1001** decent error message if it encounters a file-format version it does1002** not understand.1003**1004** bColumnsize:1005** True if the %_docsize table is created.1006**1007** bPrefixIndex:1008** This is only used for debugging. If set to false, any prefix indexes1009** are ignored. This value is configured using:1010**1011** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);1012**1013** bLocale:1014** Set to true if locale=1 was specified when the table was created.1015*/1016struct Fts5Config {1017 sqlite3 *db; /* Database handle */1018 Fts5Global *pGlobal; /* Global fts5 object for handle db */1019 char *zDb; /* Database holding FTS index (e.g. "main") */1020 char *zName; /* Name of FTS index */1021 int nCol; /* Number of columns */1022 char **azCol; /* Column names */1023 u8 *abUnindexed; /* True for unindexed columns */1024 int nPrefix; /* Number of prefix indexes */1025 int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */1026 int eContent; /* An FTS5_CONTENT value */1027 int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */1028 int bContentlessUnindexed; /* "contentless_unindexed=" option (dflt=0) */1029 char *zContent; /* content table */ 1030 char *zContentRowid; /* "content_rowid=" option value */ 1031 int bColumnsize; /* "columnsize=" option value (dflt==1) */1032 int bTokendata; /* "tokendata=" option value (dflt==0) */1033 int bLocale; /* "locale=" option value (dflt==0) */1034 int eDetail; /* FTS5_DETAIL_XXX value */1035 char *zContentExprlist;1036 Fts5TokenizerConfig t;1037 int bLock; /* True when table is preparing statement */1038 1039 1040 /* Values loaded from the %_config table */1041 int iVersion; /* fts5 file format 'version' */1042 int iCookie; /* Incremented when %_config is modified */1043 int pgsz; /* Approximate page size used in %_data */1044 int nAutomerge; /* 'automerge' setting */1045 int nCrisisMerge; /* Maximum allowed segments per level */1046 int nUsermerge; /* 'usermerge' setting */1047 int nHashSize; /* Bytes of memory for in-memory hash */1048 char *zRank; /* Name of rank function */1049 char *zRankArgs; /* Arguments to rank function */1050 int bSecureDelete; /* 'secure-delete' */1051 int nDeleteMerge; /* 'deletemerge' */1052 int bPrefixInsttoken; /* 'prefix-insttoken' */1053 1054 /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */1055 char **pzErrmsg;1056 1057#ifdef SQLITE_DEBUG1058 int bPrefixIndex; /* True to use prefix-indexes */1059#endif1060};1061 1062/* Current expected value of %_config table 'version' field. And1063** the expected version if the 'secure-delete' option has ever been1064** set on the table. */1065#define FTS5_CURRENT_VERSION 41066#define FTS5_CURRENT_VERSION_SECUREDELETE 51067 1068#define FTS5_CONTENT_NORMAL 01069#define FTS5_CONTENT_NONE 11070#define FTS5_CONTENT_EXTERNAL 21071#define FTS5_CONTENT_UNINDEXED 31072 1073#define FTS5_DETAIL_FULL 01074#define FTS5_DETAIL_NONE 11075#define FTS5_DETAIL_COLUMNS 21076 1077#define FTS5_PATTERN_NONE 01078#define FTS5_PATTERN_LIKE 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */1079#define FTS5_PATTERN_GLOB 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */1080 1081static int sqlite3Fts5ConfigParse(1082 Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**1083);1084static void sqlite3Fts5ConfigFree(Fts5Config*);1085 1086static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);1087 1088static int sqlite3Fts5Tokenize(1089 Fts5Config *pConfig, /* FTS5 Configuration object */1090 int flags, /* FTS5_TOKENIZE_* flags */1091 const char *pText, int nText, /* Text to tokenize */1092 void *pCtx, /* Context passed to xToken() */1093 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */1094);1095 1096static void sqlite3Fts5Dequote(char *z);1097 1098/* Load the contents of the %_config table */1099static int sqlite3Fts5ConfigLoad(Fts5Config*, int);1100 1101/* Set the value of a single config attribute */1102static int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);1103 1104static int sqlite3Fts5ConfigParseRank(const char*, char**, char**);1105 1106static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...);1107 1108/*1109** End of interface to code in fts5_config.c.1110**************************************************************************/1111 1112/**************************************************************************1113** Interface to code in fts5_buffer.c.1114*/1115 1116/*1117** Buffer object for the incremental building of string data.1118*/1119typedef struct Fts5Buffer Fts5Buffer;1120struct Fts5Buffer {1121 u8 *p;1122 int n;1123 int nSpace;1124};1125 1126static int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);1127static void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);1128static void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);1129static void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);1130static void sqlite3Fts5BufferFree(Fts5Buffer*);1131static void sqlite3Fts5BufferZero(Fts5Buffer*);1132static void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);1133static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);1134 1135static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);1136 1137#define fts5BufferZero(x) sqlite3Fts5BufferZero(x)1138#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c)1139#define fts5BufferFree(a) sqlite3Fts5BufferFree(a)1140#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)1141#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)1142 1143#define fts5BufferGrow(pRc,pBuf,nn) ( \1144 (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \1145 sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \1146)1147 1148/* Write and decode big-endian 32-bit integer values */1149static void sqlite3Fts5Put32(u8*, int);1150static int sqlite3Fts5Get32(const u8*);1151 1152#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF)1153#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)1154 1155typedef struct Fts5PoslistReader Fts5PoslistReader;1156struct Fts5PoslistReader {1157 /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */1158 const u8 *a; /* Position list to iterate through */1159 int n; /* Size of buffer at a[] in bytes */1160 int i; /* Current offset in a[] */1161 1162 u8 bFlag; /* For client use (any custom purpose) */1163 1164 /* Output variables */1165 u8 bEof; /* Set to true at EOF */1166 i64 iPos; /* (iCol<<32) + iPos */1167};1168static int sqlite3Fts5PoslistReaderInit(1169 const u8 *a, int n, /* Poslist buffer to iterate through */1170 Fts5PoslistReader *pIter /* Iterator object to initialize */1171);1172static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);1173 1174typedef struct Fts5PoslistWriter Fts5PoslistWriter;1175struct Fts5PoslistWriter {1176 i64 iPrev;1177};1178static int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);1179static void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);1180 1181static int sqlite3Fts5PoslistNext64(1182 const u8 *a, int n, /* Buffer containing poslist */1183 int *pi, /* IN/OUT: Offset within a[] */1184 i64 *piOff /* IN/OUT: Current offset */1185);1186 1187/* Malloc utility */1188static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte);1189static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);1190 1191/* Character set tests (like isspace(), isalpha() etc.) */1192static int sqlite3Fts5IsBareword(char t);1193 1194 1195/* Bucket of terms object used by the integrity-check in offsets=0 mode. */1196typedef struct Fts5Termset Fts5Termset;1197static int sqlite3Fts5TermsetNew(Fts5Termset**);1198static int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);1199static void sqlite3Fts5TermsetFree(Fts5Termset*);1200