AryaWu/sqlite
0
1/*2** 2014 May 313**4** The author disclaims copyright to this source code. In place of5** a legal notice, here is a blessing:6**7** May you do good and not evil.8** May you find forgiveness for yourself and forgive others.9** May you share freely, never taking more than you give.10**11******************************************************************************12**13** Interfaces to extend FTS5. Using the interfaces defined in this file, 14** FTS5 may be extended with:15**16** * custom tokenizers, and17** * custom auxiliary functions.18*/19 20 21#ifndef _FTS5_H22#define _FTS5_H23 24#include "sqlite3.h"25 26#ifdef __cplusplus27extern "C" {28#endif29 30/*************************************************************************31** CUSTOM AUXILIARY FUNCTIONS32**33** Virtual table implementations may overload SQL functions by implementing34** the sqlite3_module.xFindFunction() method.35*/36 37typedef struct Fts5ExtensionApi Fts5ExtensionApi;38typedef struct Fts5Context Fts5Context;39typedef struct Fts5PhraseIter Fts5PhraseIter;40 41typedef void (*fts5_extension_function)(42 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */43 Fts5Context *pFts, /* First arg to pass to pApi functions */44 sqlite3_context *pCtx, /* Context for returning result/error */45 int nVal, /* Number of values in apVal[] array */46 sqlite3_value **apVal /* Array of trailing arguments */47);48 49struct Fts5PhraseIter {50 const unsigned char *a;51 const unsigned char *b;52};53 54/*55** EXTENSION API FUNCTIONS56**57** xUserData(pFts):58** Return a copy of the pUserData pointer passed to the xCreateFunction()59** API when the extension function was registered.60**61** xColumnTotalSize(pFts, iCol, pnToken):62** If parameter iCol is less than zero, set output variable *pnToken63** to the total number of tokens in the FTS5 table. Or, if iCol is64** non-negative but less than the number of columns in the table, return65** the total number of tokens in column iCol, considering all rows in 66** the FTS5 table.67**68** If parameter iCol is greater than or equal to the number of columns69** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.70** an OOM condition or IO error), an appropriate SQLite error code is 71** returned.72**73** xColumnCount(pFts):74** Return the number of columns in the table.75**76** xColumnSize(pFts, iCol, pnToken):77** If parameter iCol is less than zero, set output variable *pnToken78** to the total number of tokens in the current row. Or, if iCol is79** non-negative but less than the number of columns in the table, set80** *pnToken to the number of tokens in column iCol of the current row.81**82** If parameter iCol is greater than or equal to the number of columns83** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.84** an OOM condition or IO error), an appropriate SQLite error code is 85** returned.86**87** This function may be quite inefficient if used with an FTS5 table88** created with the "columnsize=0" option.89**90** xColumnText:91** If parameter iCol is less than zero, or greater than or equal to the92** number of columns in the table, SQLITE_RANGE is returned. 93**94** Otherwise, this function attempts to retrieve the text of column iCol of95** the current document. If successful, (*pz) is set to point to a buffer96** containing the text in utf-8 encoding, (*pn) is set to the size in bytes97** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,98** if an error occurs, an SQLite error code is returned and the final values99** of (*pz) and (*pn) are undefined.100**101** xPhraseCount:102** Returns the number of phrases in the current query expression.103**104** xPhraseSize:105** If parameter iCol is less than zero, or greater than or equal to the106** number of phrases in the current query, as returned by xPhraseCount, 107** 0 is returned. Otherwise, this function returns the number of tokens in108** phrase iPhrase of the query. Phrases are numbered starting from zero.109**110** xInstCount:111** Set *pnInst to the total number of occurrences of all phrases within112** the query within the current row. Return SQLITE_OK if successful, or113** an error code (i.e. SQLITE_NOMEM) if an error occurs.114**115** This API can be quite slow if used with an FTS5 table created with the116** "detail=none" or "detail=column" option. If the FTS5 table is created 117** with either "detail=none" or "detail=column" and "content=" option 118** (i.e. if it is a contentless table), then this API always returns 0.119**120** xInst:121** Query for the details of phrase match iIdx within the current row.122** Phrase matches are numbered starting from zero, so the iIdx argument123** should be greater than or equal to zero and smaller than the value124** output by xInstCount(). If iIdx is less than zero or greater than125** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.126**127** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol128** to the column in which it occurs and *piOff the token offset of the129** first token of the phrase. SQLITE_OK is returned if successful, or an130** error code (i.e. SQLITE_NOMEM) if an error occurs.131**132** This API can be quite slow if used with an FTS5 table created with the133** "detail=none" or "detail=column" option. 134**135** xRowid:136** Returns the rowid of the current row.137**138** xTokenize:139** Tokenize text using the tokenizer belonging to the FTS5 table.140**141** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):142** This API function is used to query the FTS table for phrase iPhrase143** of the current query. Specifically, a query equivalent to:144**145** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid146**147** with $p set to a phrase equivalent to the phrase iPhrase of the148** current query is executed. Any column filter that applies to149** phrase iPhrase of the current query is included in $p. For each 150** row visited, the callback function passed as the fourth argument 151** is invoked. The context and API objects passed to the callback 152** function may be used to access the properties of each matched row.153** Invoking Api.xUserData() returns a copy of the pointer passed as 154** the third argument to pUserData.155**156** If parameter iPhrase is less than zero, or greater than or equal to157** the number of phrases in the query, as returned by xPhraseCount(),158** this function returns SQLITE_RANGE.159**160** If the callback function returns any value other than SQLITE_OK, the161** query is abandoned and the xQueryPhrase function returns immediately.162** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.163** Otherwise, the error code is propagated upwards.164**165** If the query runs to completion without incident, SQLITE_OK is returned.166** Or, if some error occurs before the query completes or is aborted by167** the callback, an SQLite error code is returned.168**169**170** xSetAuxdata(pFts5, pAux, xDelete)171**172** Save the pointer passed as the second argument as the extension function's 173** "auxiliary data". The pointer may then be retrieved by the current or any174** future invocation of the same fts5 extension function made as part of175** the same MATCH query using the xGetAuxdata() API.176**177** Each extension function is allocated a single auxiliary data slot for178** each FTS query (MATCH expression). If the extension function is invoked 179** more than once for a single FTS query, then all invocations share a 180** single auxiliary data context.181**182** If there is already an auxiliary data pointer when this function is183** invoked, then it is replaced by the new pointer. If an xDelete callback184** was specified along with the original pointer, it is invoked at this185** point.186**187** The xDelete callback, if one is specified, is also invoked on the188** auxiliary data pointer after the FTS5 query has finished.189**190** If an error (e.g. an OOM condition) occurs within this function,191** the auxiliary data is set to NULL and an error code returned. If the192** xDelete parameter was not NULL, it is invoked on the auxiliary data193** pointer before returning.194**195**196** xGetAuxdata(pFts5, bClear)197**198** Returns the current auxiliary data pointer for the fts5 extension 199** function. See the xSetAuxdata() method for details.200**201** If the bClear argument is non-zero, then the auxiliary data is cleared202** (set to NULL) before this function returns. In this case the xDelete,203** if any, is not invoked.204**205**206** xRowCount(pFts5, pnRow)207**208** This function is used to retrieve the total number of rows in the table.209** In other words, the same value that would be returned by:210**211** SELECT count(*) FROM ftstable;212**213** xPhraseFirst()214** This function is used, along with type Fts5PhraseIter and the xPhraseNext215** method, to iterate through all instances of a single query phrase within216** the current row. This is the same information as is accessible via the217** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient218** to use, this API may be faster under some circumstances. To iterate 219** through instances of phrase iPhrase, use the following code:220**221** Fts5PhraseIter iter;222** int iCol, iOff;223** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);224** iCol>=0;225** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)226** ){227** // An instance of phrase iPhrase at offset iOff of column iCol228** }229**230** The Fts5PhraseIter structure is defined above. Applications should not231** modify this structure directly - it should only be used as shown above232** with the xPhraseFirst() and xPhraseNext() API methods (and by233** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).234**235** This API can be quite slow if used with an FTS5 table created with the236** "detail=none" or "detail=column" option. If the FTS5 table is created 237** with either "detail=none" or "detail=column" and "content=" option 238** (i.e. if it is a contentless table), then this API always iterates239** through an empty set (all calls to xPhraseFirst() set iCol to -1).240**241** In all cases, matches are visited in (column ASC, offset ASC) order.242** i.e. all those in column 0, sorted by offset, followed by those in 243** column 1, etc.244**245** xPhraseNext()246** See xPhraseFirst above.247**248** xPhraseFirstColumn()249** This function and xPhraseNextColumn() are similar to the xPhraseFirst()250** and xPhraseNext() APIs described above. The difference is that instead251** of iterating through all instances of a phrase in the current row, these252** APIs are used to iterate through the set of columns in the current row253** that contain one or more instances of a specified phrase. For example:254**255** Fts5PhraseIter iter;256** int iCol;257** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);258** iCol>=0;259** pApi->xPhraseNextColumn(pFts, &iter, &iCol)260** ){261** // Column iCol contains at least one instance of phrase iPhrase262** }263**264** This API can be quite slow if used with an FTS5 table created with the265** "detail=none" option. If the FTS5 table is created with either 266** "detail=none" "content=" option (i.e. if it is a contentless table), 267** then this API always iterates through an empty set (all calls to 268** xPhraseFirstColumn() set iCol to -1).269**270** The information accessed using this API and its companion271** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext272** (or xInst/xInstCount). The chief advantage of this API is that it is273** significantly more efficient than those alternatives when used with274** "detail=column" tables. 275**276** xPhraseNextColumn()277** See xPhraseFirstColumn above.278**279** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)280** This is used to access token iToken of phrase iPhrase of the current281** query. Before returning, output parameter *ppToken is set to point282** to a buffer containing the requested token, and *pnToken to the283** size of this buffer in bytes.284**285** If iPhrase or iToken are less than zero, or if iPhrase is greater than286** or equal to the number of phrases in the query as reported by 287** xPhraseCount(), or if iToken is equal to or greater than the number of288** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken289 are both zeroed.290**291** The output text is not a copy of the query text that specified the292** token. It is the output of the tokenizer module. For tokendata=1293** tables, this includes any embedded 0x00 and trailing data.294**295** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)296** This is used to access token iToken of phrase hit iIdx within the297** current row. If iIdx is less than zero or greater than or equal to the298** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise,299** output variable (*ppToken) is set to point to a buffer containing the300** matching document token, and (*pnToken) to the size of that buffer in 301** bytes. 302**303** The output text is not a copy of the document text that was tokenized.304** It is the output of the tokenizer module. For tokendata=1 tables, this 305** includes any embedded 0x00 and trailing data.306**307** This API may be slow in some cases if the token identified by parameters 308** iIdx and iToken matched a prefix token in the query. In most cases, the309** first call to this API for each prefix token in the query is forced310** to scan the portion of the full-text index that matches the prefix311** token to collect the extra data required by this API. If the prefix312** token matches a large number of token instances in the document set,313** this may be a performance problem. 314**315** If the user knows in advance that a query may use this API for a316** prefix token, FTS5 may be configured to collect all required data as part317** of the initial querying of the full-text index, avoiding the second scan318** entirely. This also causes prefix queries that do not use this API to 319** run more slowly and use more memory. FTS5 may be configured in this way320** either on a per-table basis using the [FTS5 insttoken | 'insttoken'] 321** option, or on a per-query basis using the 322** [fts5_insttoken | fts5_insttoken()] user function.323**324** This API can be quite slow if used with an FTS5 table created with the325** "detail=none" or "detail=column" option. 326**327** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)328** If parameter iCol is less than zero, or greater than or equal to the329** number of columns in the table, SQLITE_RANGE is returned.330**331** Otherwise, this function attempts to retrieve the locale associated332** with column iCol of the current row. Usually, there is no associated333** locale, and output parameters (*pzLocale) and (*pnLocale) are set334** to NULL and 0, respectively. However, if the fts5_locale() function335** was used to associate a locale with the value when it was inserted336** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated337** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) 338** is set to the size in bytes of the buffer, not including the 339** nul-terminator.340**341** If successful, SQLITE_OK is returned. Or, if an error occurs, an342** SQLite error code is returned. The final value of the output parameters343** is undefined in this case.344**345** xTokenize_v2:346** Tokenize text using the tokenizer belonging to the FTS5 table. This347** API is the same as the xTokenize() API, except that it allows a tokenizer348** locale to be specified.349*/350struct Fts5ExtensionApi {351 int iVersion; /* Currently always set to 4 */352 353 void *(*xUserData)(Fts5Context*);354 355 int (*xColumnCount)(Fts5Context*);356 int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);357 int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);358 359 int (*xTokenize)(Fts5Context*, 360 const char *pText, int nText, /* Text to tokenize */361 void *pCtx, /* Context passed to xToken() */362 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */363 );364 365 int (*xPhraseCount)(Fts5Context*);366 int (*xPhraseSize)(Fts5Context*, int iPhrase);367 368 int (*xInstCount)(Fts5Context*, int *pnInst);369 int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);370 371 sqlite3_int64 (*xRowid)(Fts5Context*);372 int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);373 int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);374 375 int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,376 int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)377 );378 int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));379 void *(*xGetAuxdata)(Fts5Context*, int bClear);380 381 int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);382 void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);383 384 int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);385 void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);386 387 /* Below this point are iVersion>=3 only */388 int (*xQueryToken)(Fts5Context*, 389 int iPhrase, int iToken, 390 const char **ppToken, int *pnToken391 );392 int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);393 394 /* Below this point are iVersion>=4 only */395 int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);396 int (*xTokenize_v2)(Fts5Context*,397 const char *pText, int nText, /* Text to tokenize */398 const char *pLocale, int nLocale, /* Locale to pass to tokenizer */399 void *pCtx, /* Context passed to xToken() */400 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */401 );402};403 404/* 405** CUSTOM AUXILIARY FUNCTIONS406*************************************************************************/407 408/*************************************************************************409** CUSTOM TOKENIZERS410**411** Applications may also register custom tokenizer types. A tokenizer 412** is registered by providing fts5 with a populated instance of the 413** following structure. All structure methods must be defined, setting414** any member of the fts5_tokenizer struct to NULL leads to undefined415** behaviour. The structure methods are expected to function as follows:416**417** xCreate:418** This function is used to allocate and initialize a tokenizer instance.419** A tokenizer instance is required to actually tokenize text.420**421** The first argument passed to this function is a copy of the (void*)422** pointer provided by the application when the fts5_tokenizer_v2 object423** was registered with FTS5 (the third argument to xCreateTokenizer()). 424** The second and third arguments are an array of nul-terminated strings425** containing the tokenizer arguments, if any, specified following the426** tokenizer name as part of the CREATE VIRTUAL TABLE statement used427** to create the FTS5 table.428**429** The final argument is an output variable. If successful, (*ppOut) 430** should be set to point to the new tokenizer handle and SQLITE_OK431** returned. If an error occurs, some value other than SQLITE_OK should432** be returned. In this case, fts5 assumes that the final value of *ppOut 433** is undefined.434**435** xDelete:436** This function is invoked to delete a tokenizer handle previously437** allocated using xCreate(). Fts5 guarantees that this function will438** be invoked exactly once for each successful call to xCreate().439**440** xTokenize:441** This function is expected to tokenize the nText byte string indicated 442** by argument pText. pText may or may not be nul-terminated. The first443** argument passed to this function is a pointer to an Fts5Tokenizer object444** returned by an earlier call to xCreate().445**446** The third argument indicates the reason that FTS5 is requesting447** tokenization of the supplied text. This is always one of the following448** four values:449**450** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into451** or removed from the FTS table. The tokenizer is being invoked to452** determine the set of tokens to add to (or delete from) the453** FTS index.454**455** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed 456** against the FTS index. The tokenizer is being called to tokenize 457** a bareword or quoted string specified as part of the query.458**459** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as460** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is461** followed by a "*" character, indicating that the last token462** returned by the tokenizer will be treated as a token prefix.463**464** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to 465** satisfy an fts5_api.xTokenize() request made by an auxiliary466** function. Or an fts5_api.xColumnSize() request made by the same467** on a columnsize=0 database. 468** </ul>469**470** The sixth and seventh arguments passed to xTokenize() - pLocale and471** nLocale - are a pointer to a buffer containing the locale to use for472** tokenization (e.g. "en_US") and its size in bytes, respectively. The473** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in474** which case nLocale is always 0) to indicate that the tokenizer should475** use its default locale.476**477** For each token in the input string, the supplied callback xToken() must478** be invoked. The first argument to it should be a copy of the pointer479** passed as the second argument to xTokenize(). The third and fourth480** arguments are a pointer to a buffer containing the token text, and the481** size of the token in bytes. The 4th and 5th arguments are the byte offsets482** of the first byte of and first byte immediately following the text from483** which the token is derived within the input.484**485** The second argument passed to the xToken() callback ("tflags") should486** normally be set to 0. The exception is if the tokenizer supports 487** synonyms. In this case see the discussion below for details.488**489** FTS5 assumes the xToken() callback is invoked for each token in the 490** order that they occur within the input text.491**492** If an xToken() callback returns any value other than SQLITE_OK, then493** the tokenization should be abandoned and the xTokenize() method should494** immediately return a copy of the xToken() return value. Or, if the495** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,496** if an error occurs with the xTokenize() implementation itself, it497** may abandon the tokenization and return any error code other than498** SQLITE_OK or SQLITE_DONE.499**500** If the tokenizer is registered using an fts5_tokenizer_v2 object,501** then the xTokenize() method has two additional arguments - pLocale502** and nLocale. These specify the locale that the tokenizer should use503** for the current request. If pLocale and nLocale are both 0, then the504** tokenizer should use its default locale. Otherwise, pLocale points to505** an nLocale byte buffer containing the name of the locale to use as utf-8 506** text. pLocale is not nul-terminated.507**508** FTS5_TOKENIZER509**510** There is also an fts5_tokenizer object. This is an older, deprecated,511** version of fts5_tokenizer_v2. It is similar except that:512**513** <ul>514** <li> There is no "iVersion" field, and515** <li> The xTokenize() method does not take a locale argument.516** </ul>517**518** Legacy fts5_tokenizer tokenizers must be registered using the519** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().520**521** Tokenizer implementations registered using either API may be retrieved522** using both xFindTokenizer() and xFindTokenizer_v2().523**524** SYNONYM SUPPORT525**526** Custom tokenizers may also support synonyms. Consider a case in which a527** user wishes to query for a phrase such as "first place". Using the 528** built-in tokenizers, the FTS5 query 'first + place' will match instances529** of "first place" within the document set, but not alternative forms530** such as "1st place". In some applications, it would be better to match531** all instances of "first place" or "1st place" regardless of which form532** the user specified in the MATCH query text.533**534** There are several ways to approach this in FTS5:535**536** <ol><li> By mapping all synonyms to a single token. In this case, using537** the above example, this means that the tokenizer returns the538** same token for inputs "first" and "1st". Say that token is in539** fact "first", so that when the user inserts the document "I won540** 1st place" entries are added to the index for tokens "i", "won",541** "first" and "place". If the user then queries for '1st + place',542** the tokenizer substitutes "first" for "1st" and the query works543** as expected.544**545** <li> By querying the index for all synonyms of each query term546** separately. In this case, when tokenizing query text, the547** tokenizer may provide multiple synonyms for a single term 548** within the document. FTS5 then queries the index for each 549** synonym individually. For example, faced with the query:550**551** <codeblock>552** ... MATCH 'first place'</codeblock>553**554** the tokenizer offers both "1st" and "first" as synonyms for the555** first token in the MATCH query and FTS5 effectively runs a query 556** similar to:557**558** <codeblock>559** ... MATCH '(first OR 1st) place'</codeblock>560**561** except that, for the purposes of auxiliary functions, the query562** still appears to contain just two phrases - "(first OR 1st)" 563** being treated as a single phrase.564**565** <li> By adding multiple synonyms for a single term to the FTS index.566** Using this method, when tokenizing document text, the tokenizer567** provides multiple synonyms for each token. So that when a 568** document such as "I won first place" is tokenized, entries are569** added to the FTS index for "i", "won", "first", "1st" and570** "place".571**572** This way, even if the tokenizer does not provide synonyms573** when tokenizing query text (it should not - to do so would be574** inefficient), it doesn't matter if the user queries for 575** 'first + place' or '1st + place', as there are entries in the576** FTS index corresponding to both forms of the first token.577** </ol>578**579** Whether it is parsing document or query text, any call to xToken that580** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit581** is considered to supply a synonym for the previous token. For example,582** when parsing the document "I won first place", a tokenizer that supports583** synonyms would call xToken() 5 times, as follows:584**585** <codeblock>586** xToken(pCtx, 0, "i", 1, 0, 1);587** xToken(pCtx, 0, "won", 3, 2, 5);588** xToken(pCtx, 0, "first", 5, 6, 11);589** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);590** xToken(pCtx, 0, "place", 5, 12, 17);591**</codeblock>592**593** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time594** xToken() is called. Multiple synonyms may be specified for a single token595** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. 596** There is no limit to the number of synonyms that may be provided for a597** single token.598**599** In many cases, method (1) above is the best approach. It does not add 600** extra data to the FTS index or require FTS5 to query for multiple terms,601** so it is efficient in terms of disk space and query speed. However, it602** does not support prefix queries very well. If, as suggested above, the603** token "first" is substituted for "1st" by the tokenizer, then the query:604**605** <codeblock>606** ... MATCH '1s*'</codeblock>607**608** will not match documents that contain the token "1st" (as the tokenizer609** will probably not map "1s" to any prefix of "first").610**611** For full prefix support, method (3) may be preferred. In this case, 612** because the index contains entries for both "first" and "1st", prefix613** queries such as 'fi*' or '1s*' will match correctly. However, because614** extra entries are added to the FTS index, this method uses more space615** within the database.616**617** Method (2) offers a midpoint between (1) and (3). Using this method,618** a query such as '1s*' will match documents that contain the literal 619** token "1st", but not "first" (assuming the tokenizer is not able to620** provide synonyms for prefixes). However, a non-prefix query like '1st'621** will match against "1st" and "first". This method does not require622** extra disk space, as no extra entries are added to the FTS index. 623** On the other hand, it may require more CPU cycles to run MATCH queries,624** as separate queries of the FTS index are required for each synonym.625**626** When using methods (2) or (3), it is important that the tokenizer only627** provide synonyms when tokenizing document text (method (3)) or query628** text (method (2)), not both. Doing so will not cause any errors, but is629** inefficient.630*/631typedef struct Fts5Tokenizer Fts5Tokenizer;632typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;633struct fts5_tokenizer_v2 {634 int iVersion; /* Currently always 2 */635 636 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);637 void (*xDelete)(Fts5Tokenizer*);638 int (*xTokenize)(Fts5Tokenizer*, 639 void *pCtx,640 int flags, /* Mask of FTS5_TOKENIZE_* flags */641 const char *pText, int nText, 642 const char *pLocale, int nLocale,643 int (*xToken)(644 void *pCtx, /* Copy of 2nd argument to xTokenize() */645 int tflags, /* Mask of FTS5_TOKEN_* flags */646 const char *pToken, /* Pointer to buffer containing token */647 int nToken, /* Size of token in bytes */648 int iStart, /* Byte offset of token within input text */649 int iEnd /* Byte offset of end of token within input text */650 )651 );652};653 654/*655** New code should use the fts5_tokenizer_v2 type to define tokenizer656** implementations. The following type is included for legacy applications657** that still use it.658*/659typedef struct fts5_tokenizer fts5_tokenizer;660struct fts5_tokenizer {661 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);662 void (*xDelete)(Fts5Tokenizer*);663 int (*xTokenize)(Fts5Tokenizer*, 664 void *pCtx,665 int flags, /* Mask of FTS5_TOKENIZE_* flags */666 const char *pText, int nText, 667 int (*xToken)(668 void *pCtx, /* Copy of 2nd argument to xTokenize() */669 int tflags, /* Mask of FTS5_TOKEN_* flags */670 const char *pToken, /* Pointer to buffer containing token */671 int nToken, /* Size of token in bytes */672 int iStart, /* Byte offset of token within input text */673 int iEnd /* Byte offset of end of token within input text */674 )675 );676};677 678 679/* Flags that may be passed as the third argument to xTokenize() */680#define FTS5_TOKENIZE_QUERY 0x0001681#define FTS5_TOKENIZE_PREFIX 0x0002682#define FTS5_TOKENIZE_DOCUMENT 0x0004683#define FTS5_TOKENIZE_AUX 0x0008684 685/* Flags that may be passed by the tokenizer implementation back to FTS5686** as the third argument to the supplied xToken callback. */687#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */688 689/*690** END OF CUSTOM TOKENIZERS691*************************************************************************/692 693/*************************************************************************694** FTS5 EXTENSION REGISTRATION API695*/696typedef struct fts5_api fts5_api;697struct fts5_api {698 int iVersion; /* Currently always set to 3 */699 700 /* Create a new tokenizer */701 int (*xCreateTokenizer)(702 fts5_api *pApi,703 const char *zName,704 void *pUserData,705 fts5_tokenizer *pTokenizer,706 void (*xDestroy)(void*)707 );708 709 /* Find an existing tokenizer */710 int (*xFindTokenizer)(711 fts5_api *pApi,712 const char *zName,713 void **ppUserData,714 fts5_tokenizer *pTokenizer715 );716 717 /* Create a new auxiliary function */718 int (*xCreateFunction)(719 fts5_api *pApi,720 const char *zName,721 void *pUserData,722 fts5_extension_function xFunction,723 void (*xDestroy)(void*)724 );725 726 /* APIs below this point are only available if iVersion>=3 */727 728 /* Create a new tokenizer */729 int (*xCreateTokenizer_v2)(730 fts5_api *pApi,731 const char *zName,732 void *pUserData,733 fts5_tokenizer_v2 *pTokenizer,734 void (*xDestroy)(void*)735 );736 737 /* Find an existing tokenizer */738 int (*xFindTokenizer_v2)(739 fts5_api *pApi,740 const char *zName,741 void **ppUserData,742 fts5_tokenizer_v2 **ppTokenizer743 );744};745 746/*747** END OF REGISTRATION API748*************************************************************************/749 750#ifdef __cplusplus751} /* end of the 'extern "C"' block */752#endif753 754#endif /* _FTS5_H */755