AryaWu/sqlite
0
1/*2** 2015-05-253**4** The author disclaims copyright to this source code. In place of5** a legal notice, here is a blessing:6**7** May you do good and not evil.8** May you find forgiveness for yourself and forgive others.9** May you share freely, never taking more than you give.10**11*************************************************************************12**13** This is a utility program designed to aid running regressions tests on14** the SQLite library using data from external fuzzers.15**16** This program reads content from an SQLite database file with the following17** schema:18**19** CREATE TABLE db(20** dbid INTEGER PRIMARY KEY, -- database id21** dbcontent BLOB -- database disk file image22** );23** CREATE TABLE xsql(24** sqlid INTEGER PRIMARY KEY, -- SQL script id25** sqltext TEXT -- Text of SQL statements to run26** );27** CREATE TABLE IF NOT EXISTS readme(28** msg TEXT -- Human-readable description of this test collection29** );30**31** For each database file in the DB table, the SQL text in the XSQL table32** is run against that database. All README.MSG values are printed prior33** to the start of the test (unless the --quiet option is used). If the34** DB table is empty, then all entries in XSQL are run against an empty35** in-memory database.36**37** This program is looking for crashes, assertion faults, and/or memory leaks.38** No attempt is made to verify the output. The assumption is that either all39** of the database files or all of the SQL statements are malformed inputs,40** generated by a fuzzer, that need to be checked to make sure they do not41** present a security risk.42**43** This program also includes some command-line options to help with 44** creation and maintenance of the source content database. The command45**46** ./fuzzcheck database.db --load-sql FILE...47**48** Loads all FILE... arguments into the XSQL table. The --load-db option49** works the same but loads the files into the DB table. The -m option can50** be used to initialize the README table. The "database.db" file is created51** if it does not previously exist. Example:52**53** ./fuzzcheck new.db --load-sql *.sql54** ./fuzzcheck new.db --load-db *.db55** ./fuzzcheck new.db -m 'New test cases'56**57** The three commands above will create the "new.db" file and initialize all58** tables. Then do "./fuzzcheck new.db" to run the tests.59**60** DEBUGGING HINTS:61**62** If fuzzcheck does crash, it can be run in the debugger and the content63** of the global variable g.zTextName[] will identify the specific XSQL and64** DB values that were running when the crash occurred.65**66** DBSQLFUZZ: (Added 2020-02-25)67**68** The dbsqlfuzz fuzzer includes both a database file and SQL to run against69** that database in its input. This utility can now process dbsqlfuzz70** input files. Load such files using the "--load-dbsql FILE ..." command-line71** option.72**73** Dbsqlfuzz inputs are ordinary text. The first part of the file is text74** that describes the content of the database (using a lot of hexadecimal),75** then there is a divider line followed by the SQL to run against the76** database. Because they are ordinary text, dbsqlfuzz inputs are stored77** in the XSQL table, as if they were ordinary SQL inputs. The isDbSql()78** function can look at a text string and determine whether or not it is79** a valid dbsqlfuzz input.80*/81#include <stdio.h>82#include <stdlib.h>83#include <string.h>84#include <stdarg.h>85#include <ctype.h>86#include <assert.h>87#include "sqlite3.h"88#include "sqlite3recover.h"89#define ISSPACE(X) isspace((unsigned char)(X))90#define ISDIGIT(X) isdigit((unsigned char)(X))91#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)92# define FLEXARRAY93#else94# define FLEXARRAY 195#endif96 97 98#ifdef __unix__99# include <signal.h>100# include <unistd.h>101#endif102 103#include <stddef.h>104#if !defined(_MSC_VER)105# include <stdint.h>106#endif107 108#if defined(_MSC_VER)109typedef unsigned char uint8_t;110#endif111 112/*113** Files in the virtual file system.114*/115typedef struct VFile VFile;116struct VFile {117 char *zFilename; /* Filename. NULL for delete-on-close. From malloc() */118 int sz; /* Size of the file in bytes */119 int nRef; /* Number of references to this file */120 unsigned char *a; /* Content of the file. From malloc() */121};122typedef struct VHandle VHandle;123struct VHandle {124 sqlite3_file base; /* Base class. Must be first */125 VFile *pVFile; /* The underlying file */126};127 128/*129** The value of a database file template, or of an SQL script130*/131typedef struct Blob Blob;132struct Blob {133 Blob *pNext; /* Next in a list */134 int id; /* Id of this Blob */135 int seq; /* Sequence number */136 int sz; /* Size of this Blob in bytes */137 unsigned char a[FLEXARRAY]; /* Blob content. Allocated as needed. */138};139 140/* Size in bytes of a Blob object sufficient to store N byte of content */141#define SZ_BLOB(N) (offsetof(Blob,a) + (((N)+7)&~7))142 143/*144** Maximum number of files in the in-memory virtual filesystem.145*/146#define MX_FILE 10147 148/*149** Maximum allowed file size150*/151#define MX_FILE_SZ 10000000152 153/*154** All global variables are gathered into the "g" singleton.155*/156static struct GlobalVars {157 const char *zArgv0; /* Name of program */158 const char *zDbFile; /* Name of database file */159 VFile aFile[MX_FILE]; /* The virtual filesystem */160 int nDb; /* Number of template databases */161 Blob *pFirstDb; /* Content of first template database */162 int nSql; /* Number of SQL scripts */163 Blob *pFirstSql; /* First SQL script */164 unsigned int uRandom; /* Seed for the SQLite PRNG */165 unsigned int nInvariant; /* Number of invariant checks run */166 char zTestName[100]; /* Name of current test */167} g;168 169/*170** Include various extensions.171*/172extern int sqlite3_vt02_init(sqlite3*,char**,const sqlite3_api_routines*);173extern int sqlite3_randomjson_init(sqlite3*,char**,const sqlite3_api_routines*);174 175/*176** Print an error message and quit.177*/178static void fatalError(const char *zFormat, ...){179 va_list ap;180 fprintf(stderr, "%s", g.zArgv0);181 if( g.zDbFile ) fprintf(stderr, " %s", g.zDbFile);182 if( g.zTestName[0] ) fprintf(stderr, " (%s)", g.zTestName);183 fprintf(stderr, ": ");184 va_start(ap, zFormat);185 vfprintf(stderr, zFormat, ap);186 va_end(ap);187 fprintf(stderr, "\n");188 exit(1);189}190 191/*192** signal handler193*/194#ifdef __unix__195static void signalHandler(int signum){196 const char *zSig;197 if( signum==SIGABRT ){198 zSig = "abort";199 }else if( signum==SIGALRM ){200 zSig = "timeout";201 }else if( signum==SIGSEGV ){202 zSig = "segfault";203 }else{204 zSig = "signal";205 }206 fatalError(zSig);207}208#endif209 210/*211** Set the an alarm to go off after N seconds. Disable the alarm212** if N==0213*/214static void setAlarm(int N){215#ifdef __unix__216 alarm(N);217#else218 (void)N;219#endif220}221 222#ifndef SQLITE_OMIT_PROGRESS_CALLBACK223/*224** This an SQL progress handler. After an SQL statement has run for225** many steps, we want to interrupt it. This guards against infinite226** loops from recursive common table expressions.227**228** *pVdbeLimitFlag is true if the --limit-vdbe command-line option is used.229** In that case, hitting the progress handler is a fatal error.230*/231static int progressHandler(void *pVdbeLimitFlag){232 if( *(int*)pVdbeLimitFlag ) fatalError("too many VDBE cycles");233 return 1;234}235#endif236 237/*238** Reallocate memory. Show an error and quit if unable.239*/240static void *safe_realloc(void *pOld, int szNew){241 void *pNew = realloc(pOld, szNew<=0 ? 1 : szNew);242 if( pNew==0 ) fatalError("unable to realloc for %d bytes", szNew);243 return pNew;244}245 246/*247** Initialize the virtual file system.248*/249static void formatVfs(void){250 int i;251 for(i=0; i<MX_FILE; i++){252 g.aFile[i].sz = -1;253 g.aFile[i].zFilename = 0;254 g.aFile[i].a = 0;255 g.aFile[i].nRef = 0;256 }257}258 259 260/*261** Erase all information in the virtual file system.262*/263static void reformatVfs(void){264 int i;265 for(i=0; i<MX_FILE; i++){266 if( g.aFile[i].sz<0 ) continue;267 if( g.aFile[i].zFilename ){268 free(g.aFile[i].zFilename);269 g.aFile[i].zFilename = 0;270 }271 if( g.aFile[i].nRef>0 ){272 fatalError("file %d still open. nRef=%d", i, g.aFile[i].nRef);273 }274 g.aFile[i].sz = -1;275 free(g.aFile[i].a);276 g.aFile[i].a = 0;277 g.aFile[i].nRef = 0;278 }279}280 281/*282** Find a VFile by name283*/284static VFile *findVFile(const char *zName){285 int i;286 if( zName==0 ) return 0;287 for(i=0; i<MX_FILE; i++){288 if( g.aFile[i].zFilename==0 ) continue; 289 if( strcmp(g.aFile[i].zFilename, zName)==0 ) return &g.aFile[i];290 }291 return 0;292}293 294/*295** Find a VFile by name. Create it if it does not already exist and296** initialize it to the size and content given.297**298** Return NULL only if the filesystem is full.299*/300static VFile *createVFile(const char *zName, int sz, unsigned char *pData){301 VFile *pNew = findVFile(zName);302 int i;303 if( pNew ) return pNew;304 for(i=0; i<MX_FILE && g.aFile[i].sz>=0; i++){}305 if( i>=MX_FILE ) return 0;306 pNew = &g.aFile[i];307 if( zName ){308 int nName = (int)strlen(zName)+1;309 pNew->zFilename = safe_realloc(0, nName);310 memcpy(pNew->zFilename, zName, nName);311 }else{312 pNew->zFilename = 0;313 }314 pNew->nRef = 0;315 pNew->sz = sz;316 pNew->a = safe_realloc(0, sz);317 if( sz>0 ) memcpy(pNew->a, pData, sz);318 return pNew;319}320 321/* Return true if the line is all zeros */322static int allZero(unsigned char *aLine){323 int i;324 for(i=0; i<16 && aLine[i]==0; i++){}325 return i==16;326}327 328/*329** Render a database and query as text that can be input into330** the CLI.331*/332static void renderDbSqlForCLI(333 FILE *out, /* Write to this file */334 const char *zFile, /* Name of the database file */335 unsigned char *aDb, /* Database content */336 int nDb, /* Number of bytes in aDb[] */337 unsigned char *zSql, /* SQL content */338 int nSql /* Bytes of SQL */339){340 fprintf(out, ".print ******* %s *******\n", zFile);341 if( nDb>100 ){342 int i, j; /* Loop counters */343 int pgsz; /* Size of each page */344 int lastPage = 0; /* Last page number shown */345 int iPage; /* Current page number */346 unsigned char *aLine; /* Single line to display */347 unsigned char buf[16]; /* Fake line */348 unsigned char bShow[256]; /* Characters ok to display */349 350 memset(bShow, '.', sizeof(bShow));351 for(i=' '; i<='~'; i++){352 if( i!='{' && i!='}' && i!='"' && i!='\\' ) bShow[i] = i;353 }354 pgsz = (aDb[16]<<8) | aDb[17];355 if( pgsz==0 ) pgsz = 65536;356 if( pgsz<512 || (pgsz&(pgsz-1))!=0 ) pgsz = 4096;357 fprintf(out,".open --hexdb\n");358 fprintf(out,"| size %d pagesize %d filename %s\n",nDb,pgsz,zFile);359 for(i=0; i<nDb; i += 16){360 if( i+16>nDb ){361 memset(buf, 0, sizeof(buf));362 memcpy(buf, aDb+i, nDb-i);363 aLine = buf;364 }else{365 aLine = aDb + i;366 }367 if( allZero(aLine) ) continue;368 iPage = i/pgsz + 1;369 if( lastPage!=iPage ){370 fprintf(out,"| page %d offset %d\n", iPage, (iPage-1)*pgsz);371 lastPage = iPage;372 }373 fprintf(out,"| %5d:", i-(iPage-1)*pgsz);374 for(j=0; j<16; j++) fprintf(out," %02x", aLine[j]);375 fprintf(out," ");376 for(j=0; j<16; j++){377 unsigned char c = (unsigned char)aLine[j];378 fputc( bShow[c], stdout);379 }380 fputc('\n', stdout);381 }382 fprintf(out,"| end %s\n", zFile);383 }else{384 fprintf(out,".open :memory:\n");385 }386 fprintf(out,".testctrl prng_seed 1 db\n");387 fprintf(out,".testctrl internal_functions\n");388 fprintf(out,"%.*s", nSql, zSql);389 if( nSql>0 && zSql[nSql-1]!='\n' ) fprintf(out, "\n");390}391 392/*393** Find the tail (the last component) of a pathname.394*/395static const char *pathTail(const char *zPath){396 const char *zTail = zPath;397 while( zPath[0] ){398 if( zPath[0]=='/' && zPath[1]!=0 ) zTail = &zPath[1];399#ifndef __unix__400 if( zPath[0]=='\\' && zPath[1]!=0 ) zTail = &zPath[1];401#endif402 zPath++;403 }404 return zTail;405}406 407/*408** Read the complete content of a file into memory. Add a 0x00 terminator409** and return a pointer to the result.410**411** The file content is held in memory obtained from sqlite_malloc64() which412** should be freed by the caller.413*/414static char *readFile(const char *zFilename, long *sz){415 FILE *in;416 long nIn;417 unsigned char *pBuf;418 419 *sz = 0;420 if( zFilename==0 ) return 0;421 in = fopen(zFilename, "rb");422 if( in==0 ) return 0;423 fseek(in, 0, SEEK_END);424 *sz = nIn = ftell(in);425 rewind(in);426 pBuf = sqlite3_malloc64( nIn+1 );427 if( pBuf && 1==fread(pBuf, nIn, 1, in) ){428 pBuf[nIn] = 0;429 fclose(in);430 return (char*)pBuf;431 } 432 sqlite3_free(pBuf);433 *sz = 0;434 fclose(in);435 return 0;436}437 438 439/*440** Implementation of the "readfile(X)" SQL function. The entire content441** of the file named X is read and returned as a BLOB. NULL is returned442** if the file does not exist or is unreadable.443*/444static void readfileFunc(445 sqlite3_context *context,446 int argc,447 sqlite3_value **argv448){449 long nIn;450 void *pBuf;451 const char *zName = (const char*)sqlite3_value_text(argv[0]);452 453 if( zName==0 ) return;454 pBuf = readFile(zName, &nIn);455 if( pBuf ){456 sqlite3_result_blob(context, pBuf, nIn, sqlite3_free);457 }458}459 460/*461** Implementation of the "readtextfile(X)" SQL function. The text content462** of the file named X through the end of the file or to the first \000463** character, whichever comes first, is read and returned as TEXT. NULL464** is returned if the file does not exist or is unreadable.465*/466static void readtextfileFunc(467 sqlite3_context *context,468 int argc,469 sqlite3_value **argv470){471 const char *zName;472 FILE *in;473 long nIn;474 char *pBuf;475 476 zName = (const char*)sqlite3_value_text(argv[0]);477 if( zName==0 ) return;478 in = fopen(zName, "rb");479 if( in==0 ) return;480 fseek(in, 0, SEEK_END);481 nIn = ftell(in);482 rewind(in);483 pBuf = sqlite3_malloc64( nIn+1 );484 if( pBuf && 1==fread(pBuf, nIn, 1, in) ){485 pBuf[nIn] = 0;486 sqlite3_result_text(context, pBuf, -1, sqlite3_free);487 }else{488 sqlite3_free(pBuf);489 }490 fclose(in);491}492 493/*494** Implementation of the "writefile(X,Y)" SQL function. The argument Y495** is written into file X. The number of bytes written is returned. Or496** NULL is returned if something goes wrong, such as being unable to open497** file X for writing.498*/499static void writefileFunc(500 sqlite3_context *context,501 int argc,502 sqlite3_value **argv503){504 FILE *out;505 const char *z;506 sqlite3_int64 rc;507 const char *zFile;508 509 (void)argc;510 zFile = (const char*)sqlite3_value_text(argv[0]);511 if( zFile==0 ) return;512 out = fopen(zFile, "wb");513 if( out==0 ) return;514 z = (const char*)sqlite3_value_blob(argv[1]);515 if( z==0 ){516 rc = 0;517 }else{518 rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);519 }520 fclose(out);521 sqlite3_result_int64(context, rc);522}523 524 525/*526** Load a list of Blob objects from the database527*/528static void blobListLoadFromDb(529 sqlite3 *db, /* Read from this database */530 const char *zSql, /* Query used to extract the blobs */531 int firstId, /* First sqlid to load */532 int lastId, /* Last sqlid to load */533 int *pN, /* OUT: Write number of blobs loaded here */534 Blob **ppList /* OUT: Write the head of the blob list here */535){536 Blob *head;537 Blob *p;538 sqlite3_stmt *pStmt;539 int n = 0;540 int rc;541 char *z2;542 union {543 Blob sBlob;544 unsigned char tmp[SZ_BLOB(8)];545 } uBlob;546 547 head = &uBlob.sBlob;548 if( firstId>0 ){549 z2 = sqlite3_mprintf("%s WHERE rowid BETWEEN %d AND %d", zSql,550 firstId, lastId);551 }else{552 z2 = sqlite3_mprintf("%s", zSql);553 }554 rc = sqlite3_prepare_v2(db, z2, -1, &pStmt, 0);555 sqlite3_free(z2);556 if( rc ) fatalError("%s", sqlite3_errmsg(db));557 head->pNext = 0;558 p = head;559 while( SQLITE_ROW==sqlite3_step(pStmt) ){560 int sz = sqlite3_column_bytes(pStmt, 1);561 Blob *pNew = safe_realloc(0, SZ_BLOB(sz+1));562 pNew->id = sqlite3_column_int(pStmt, 0);563 pNew->sz = sz;564 pNew->seq = n++;565 pNew->pNext = 0;566 memcpy(pNew->a, sqlite3_column_blob(pStmt,1), sz);567 pNew->a[sz] = 0;568 p->pNext = pNew;569 p = pNew;570 }571 sqlite3_finalize(pStmt);572 *pN = n;573 *ppList = head->pNext;574}575 576/*577** Free a list of Blob objects578*/579static void blobListFree(Blob *p){580 Blob *pNext;581 while( p ){582 pNext = p->pNext;583 free(p);584 p = pNext;585 }586}587 588/* Return the current wall-clock time589**590** The number of milliseconds since the julian epoch.591** 1907-01-01 00:00:00 -> 210866716800000592** 2021-01-01 00:00:00 -> 212476176000000593*/594static sqlite3_int64 timeOfDay(void){595 static sqlite3_vfs *clockVfs = 0;596 sqlite3_int64 t;597 if( clockVfs==0 ){598 clockVfs = sqlite3_vfs_find(0);599 if( clockVfs==0 ) return 0;600 }601 if( clockVfs->iVersion>=1 && clockVfs->xCurrentTimeInt64!=0 ){602 clockVfs->xCurrentTimeInt64(clockVfs, &t);603 }else{604 double r;605 clockVfs->xCurrentTime(clockVfs, &r);606 t = (sqlite3_int64)(r*86400000.0);607 }608 return t;609}610 611/***************************************************************************612** Code to process combined database+SQL scripts generated by the613** dbsqlfuzz fuzzer.614*/615 616/* An instance of the following object is passed by pointer as the617** client data to various callbacks.618*/619typedef struct FuzzCtx {620 sqlite3 *db; /* The database connection */621 sqlite3_int64 iCutoffTime; /* Stop processing at this time. */622 sqlite3_int64 iLastCb; /* Time recorded for previous progress callback */623 sqlite3_int64 mxInterval; /* Longest interval between two progress calls */624 unsigned nCb; /* Number of progress callbacks */625 unsigned mxCb; /* Maximum number of progress callbacks allowed */626 unsigned execCnt; /* Number of calls to the sqlite3_exec callback */627 int timeoutHit; /* True when reaching a timeout */628} FuzzCtx;629 630/* Verbosity level for the dbsqlfuzz test runner */631static int eVerbosity = 0;632 633/* True to activate PRAGMA vdbe_debug=on */634static int bVdbeDebug = 0;635 636/* Timeout for each fuzzing attempt, in milliseconds */637static int giTimeout = 10000; /* Defaults to 10 seconds */638 639/* Maximum number of progress handler callbacks */640static unsigned int mxProgressCb = 2000;641 642/* Maximum string length in SQLite */643static int lengthLimit = 1000000;644 645/* Maximum expression depth */646static int depthLimit = 500;647 648/* Limit on the amount of heap memory that can be used */649static sqlite3_int64 heapLimit = 100000000;650 651/* Maximum byte-code program length in SQLite */652static int vdbeOpLimit = 25000;653 654/* Maximum size of the in-memory database */655static sqlite3_int64 maxDbSize = 104857600;656/* OOM simulation parameters */657static unsigned int oomCounter = 0; /* Simulate OOM when equals 1 */658static unsigned int oomRepeat = 0; /* Number of OOMs in a row */659static void*(*defaultMalloc)(int) = 0; /* The low-level malloc routine */660 661/* Enable recovery */662static int bNoRecover = 0;663 664/* This routine is called when a simulated OOM occurs. It is broken665** out as a separate routine to make it easy to set a breakpoint on666** the OOM667*/668void oomFault(void){669 if( eVerbosity ){670 printf("Simulated OOM fault\n");671 }672 if( oomRepeat>0 ){673 oomRepeat--;674 }else{675 oomCounter--;676 }677}678 679/* This routine is a replacement malloc() that is used to simulate680** Out-Of-Memory (OOM) errors for testing purposes.681*/682static void *oomMalloc(int nByte){683 if( oomCounter ){684 if( oomCounter==1 ){685 oomFault();686 return 0;687 }else{688 oomCounter--;689 }690 }691 return defaultMalloc(nByte);692}693 694/* Register the OOM simulator. This must occur before any memory695** allocations */696static void registerOomSimulator(void){697 sqlite3_mem_methods mem;698 sqlite3_shutdown();699 sqlite3_config(SQLITE_CONFIG_GETMALLOC, &mem);700 defaultMalloc = mem.xMalloc;701 mem.xMalloc = oomMalloc;702 sqlite3_config(SQLITE_CONFIG_MALLOC, &mem);703}704 705/* Turn off any pending OOM simulation */706static void disableOom(void){707 oomCounter = 0;708 oomRepeat = 0;709}710 711/*712** Translate a single byte of Hex into an integer.713** This routine only works if h really is a valid hexadecimal714** character: 0..9a..fA..F715*/716static unsigned char hexToInt(unsigned int h){717#ifdef SQLITE_EBCDIC718 h += 9*(1&~(h>>4)); /* EBCDIC */719#else720 h += 9*(1&(h>>6)); /* ASCII */721#endif722 return h & 0xf;723}724 725/*726** The first character of buffer zIn[0..nIn-1] is a '['. This routine727** checked to see if the buffer holds "[NNNN]" or "[+NNNN]" and if it728** does it makes corresponding changes to the *pK value and *pI value729** and returns true. If the input buffer does not match the patterns,730** no changes are made to either *pK or *pI and this routine returns false.731*/732static int isOffset(733 const unsigned char *zIn, /* Text input */734 int nIn, /* Bytes of input */735 unsigned int *pK, /* half-byte cursor to adjust */736 unsigned int *pI /* Input index to adjust */737){738 int i;739 unsigned int k = 0;740 unsigned char c;741 for(i=1; i<nIn && (c = zIn[i])!=']'; i++){742 if( !isxdigit(c) ) return 0;743 k = k*16 + hexToInt(c);744 }745 if( i==nIn ) return 0;746 *pK = 2*k;747 *pI += i;748 return 1;749}750 751/*752** Decode the text starting at zIn into a binary database file.753** The maximum length of zIn is nIn bytes. Store the binary database754** file in space obtained from sqlite3_malloc().755**756** Return the number of bytes of zIn consumed. Or return -1 if there757** is an error. One potential error is that the recipe specifies a758** database file larger than MX_FILE_SZ bytes.759**760** Abort on an OOM.761*/762static int decodeDatabase(763 const unsigned char *zIn, /* Input text to be decoded */764 int nIn, /* Bytes of input text */765 unsigned char **paDecode, /* OUT: decoded database file */766 int *pnDecode /* OUT: Size of decoded database */767){768 unsigned char *a, *aNew; /* Database under construction */769 int mx = 0; /* Current size of the database */770 sqlite3_uint64 nAlloc = 4096; /* Space allocated in a[] */771 unsigned int i; /* Next byte of zIn[] to read */772 unsigned int j; /* Temporary integer */773 unsigned int k; /* half-byte cursor index for output */774 unsigned int n; /* Number of bytes of input */775 unsigned char b = 0;776 if( nIn<4 ) return -1;777 n = (unsigned int)nIn;778 a = sqlite3_malloc64( nAlloc );779 if( a==0 ){780 fprintf(stderr, "Out of memory!\n");781 exit(1);782 }783 memset(a, 0, (size_t)nAlloc);784 for(i=k=0; i<n; i++){785 unsigned char c = (unsigned char)zIn[i];786 if( isxdigit(c) ){787 k++;788 if( k & 1 ){789 b = hexToInt(c)*16;790 }else{791 b += hexToInt(c);792 j = k/2 - 1;793 if( j>=nAlloc ){794 sqlite3_uint64 newSize;795 if( nAlloc==MX_FILE_SZ || j>=MX_FILE_SZ ){796 if( eVerbosity ){797 fprintf(stderr, "Input database too big: max %d bytes\n",798 MX_FILE_SZ);799 }800 sqlite3_free(a);801 return -1;802 }803 newSize = nAlloc*2;804 if( newSize<=j ){805 newSize = (j+4096)&~4095;806 }807 if( newSize>MX_FILE_SZ ){808 if( j>=MX_FILE_SZ ){809 sqlite3_free(a);810 return -1;811 }812 newSize = MX_FILE_SZ;813 }814 aNew = sqlite3_realloc64( a, newSize );815 if( aNew==0 ){816 sqlite3_free(a);817 return -1;818 }819 a = aNew;820 assert( newSize > nAlloc );821 memset(a+nAlloc, 0, (size_t)(newSize - nAlloc));822 nAlloc = newSize;823 }824 if( j>=(unsigned)mx ){825 mx = (j + 4095)&~4095;826 if( mx>MX_FILE_SZ ) mx = MX_FILE_SZ;827 }828 assert( j<nAlloc );829 a[j] = b;830 }831 }else if( zIn[i]=='[' && i<n-3 && isOffset(zIn+i, nIn-i, &k, &i) ){832 continue;833 }else if( zIn[i]=='\n' && i<n-4 && memcmp(zIn+i,"\n--\n",4)==0 ){834 i += 4;835 break;836 }837 }838 *pnDecode = mx;839 *paDecode = a;840 return i;841}842 843/*844** Progress handler callback.845**846** The argument is the cutoff-time after which all processing should847** stop. So return non-zero if the cut-off time is exceeded.848*/849static int progress_handler(void *pClientData) {850 FuzzCtx *p = (FuzzCtx*)pClientData;851 sqlite3_int64 iNow = timeOfDay();852 int rc = iNow>=p->iCutoffTime;853 sqlite3_int64 iDiff = iNow - p->iLastCb;854 /* printf("time-remaining: %lld\n", p->iCutoffTime - iNow); */855 if( iDiff > p->mxInterval ) p->mxInterval = iDiff;856 p->nCb++;857 if( rc==0 && p->mxCb>0 && p->mxCb<=p->nCb ) rc = 1;858 if( rc && !p->timeoutHit && eVerbosity>=2 ){859 printf("Timeout on progress callback %d\n", p->nCb);860 fflush(stdout);861 p->timeoutHit = 1;862 }863 return rc;864}865 866/*867** Flag bits set by block_troublesome_sql()868*/869#define BTS_SELECT 0x000001870#define BTS_NONSELECT 0x000002871#define BTS_BADFUNC 0x000004872#define BTS_BADPRAGMA 0x000008 /* Sticky for rest of the script */873 874/*875** Disallow debugging pragmas such as "PRAGMA vdbe_debug" and876** "PRAGMA parser_trace" since they can dramatically increase the877** amount of output without actually testing anything useful.878**879** Also block ATTACH if attaching a file from the filesystem.880*/881static int block_troublesome_sql(882 void *pClientData,883 int eCode,884 const char *zArg1,885 const char *zArg2,886 const char *zArg3,887 const char *zArg4888){889 unsigned int *pBtsFlags = (unsigned int*)pClientData;890 891 (void)zArg3;892 (void)zArg4;893 switch( eCode ){894 case SQLITE_PRAGMA: {895 if( sqlite3_stricmp("busy_timeout",zArg1)==0896 && (zArg2==0 || strtoll(zArg2,0,0)>100 || strtoll(zArg2,0,10)>100)897 ){898 return SQLITE_DENY;899 }else if( sqlite3_stricmp("hard_heap_limit", zArg1)==0900 || sqlite3_stricmp("reverse_unordered_selects", zArg1)==0901 ){902 /* BTS_BADPRAGMA is sticky. A hard_heap_limit or903 ** revert_unordered_selects should inhibit all future attempts904 ** at verifying query invariants */905 *pBtsFlags |= BTS_BADPRAGMA;906 }else if( eVerbosity==0 ){907 if( sqlite3_strnicmp("vdbe_", zArg1, 5)==0908 || sqlite3_stricmp("parser_trace", zArg1)==0909 || sqlite3_stricmp("temp_store_directory", zArg1)==0910 ){911 return SQLITE_DENY;912 }913 }else if( sqlite3_stricmp("oom",zArg1)==0914 && zArg2!=0 && zArg2[0]!=0 ){915 oomCounter = atoi(zArg2);916 }917 *pBtsFlags |= BTS_NONSELECT;918 break;919 }920 case SQLITE_ATTACH: {921 /* Deny the ATTACH if it is attaching anything other than an in-memory922 ** database. */923 *pBtsFlags |= BTS_NONSELECT;924 if( zArg1==0 ) return SQLITE_DENY;925 if( strcmp(zArg1,":memory:")==0 ) return SQLITE_OK;926 if( sqlite3_strglob("file:*[?]vfs=memdb", zArg1)==0927 && sqlite3_strglob("file:*[^/a-zA-Z0-9_.]*[?]vfs=memdb", zArg1)!=0928 ){929 return SQLITE_OK;930 }931 return SQLITE_DENY;932 }933 case SQLITE_SELECT: {934 *pBtsFlags |= BTS_SELECT;935 break;936 }937 case SQLITE_FUNCTION: {938 static const char *azBadFuncs[] = {939 "avg",940 "count",941 "cume_dist",942 "current_date",943 "current_time",944 "current_timestamp",945 "date",946 "datetime",947 "decimal_sum",948 "dense_rank",949 "first_value",950 "geopoly_group_bbox",951 "group_concat",952 "implies_nonnull_row",953 "json_group_array",954 "json_group_object",955 "julianday",956 "lag",957 "last_value",958 "lead",959 "max",960 "min",961 "nth_value",962 "ntile",963 "percent_rank",964 "random",965 "randomblob",966 "rank",967 "row_number",968 "sqlite_offset",969 "strftime",970 "sum",971 "time",972 "total",973 "unixepoch",974 };975 int first, last;976 first = 0;977 last = sizeof(azBadFuncs)/sizeof(azBadFuncs[0]) - 1;978 do{979 int mid = (first+last)/2;980 int c = sqlite3_stricmp(azBadFuncs[mid], zArg2);981 if( c<0 ){982 first = mid+1;983 }else if( c>0 ){984 last = mid-1;985 }else{986 *pBtsFlags |= BTS_BADFUNC;987 break;988 }989 }while( first<=last );990 break;991 }992 case SQLITE_READ: {993 /* Benign */994 break;995 }996 default: {997 *pBtsFlags |= BTS_NONSELECT;998 }999 }1000 return SQLITE_OK;1001}1002 1003/* Implementation found in fuzzinvariant.c */1004extern int fuzz_invariant(1005 sqlite3 *db, /* The database connection */1006 sqlite3_stmt *pStmt, /* Test statement stopped on an SQLITE_ROW */1007 int iCnt, /* Invariant sequence number, starting at 0 */1008 int iRow, /* The row number for pStmt */1009 int nRow, /* Total number of output rows */1010 int *pbCorrupt, /* IN/OUT: Flag indicating a corrupt database file */1011 int eVerbosity, /* How much debugging output */1012 unsigned int dbOpt /* Default optimization flags */1013);1014 1015/* Implementation of sqlite_dbdata and sqlite_dbptr */1016extern int sqlite3_dbdata_init(sqlite3*,const char**,void*);1017 1018 1019/*1020** This function is used as a callback by the recover extension. Simply1021** print the supplied SQL statement to stdout.1022*/1023static int recoverSqlCb(void *pCtx, const char *zSql){1024 if( eVerbosity>=2 && zSql ){1025 printf("%s\n", zSql);1026 }1027 return SQLITE_OK;1028}1029 1030/*1031** This function is called to recover data from the database.1032*/1033static int recoverDatabase(sqlite3 *db){1034 int rc; /* Return code from this routine */1035 const char *zRecoveryDb = ""; /* Name of "recovery" database */1036 const char *zLAF = "lost_and_found"; /* Name of "lost_and_found" table */1037 int bFreelist = 1; /* True to scan the freelist */1038 int bRowids = 1; /* True to restore ROWID values */1039 sqlite3_recover *p = 0; /* The recovery object */1040 1041 p = sqlite3_recover_init_sql(db, "main", recoverSqlCb, 0);1042 sqlite3_recover_config(p, 789, (void*)zRecoveryDb);1043 sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF);1044 sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids);1045 sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist);1046 sqlite3_recover_run(p);1047 if( sqlite3_recover_errcode(p)!=SQLITE_OK ){1048 const char *zErr = sqlite3_recover_errmsg(p);1049 int errCode = sqlite3_recover_errcode(p);1050 if( eVerbosity>0 ){1051 printf("recovery error: %s (%d)\n", zErr, errCode);1052 }1053 }1054 rc = sqlite3_recover_finish(p);1055 if( eVerbosity>0 && rc ){1056 printf("recovery returns error code %d\n", rc);1057 }1058 return rc;1059}1060 1061/*1062** Special parameter binding, for testing and debugging purposes.1063**1064** $int_NNN -> integer value NNN1065** $text_TTTT -> floating point value TTT with destructor1066** $carray_clr -> First argument to carray() for color names1067** $carray_primes -> First argument to carray() for prime numbers1068*/1069static void bindDebugParameters(sqlite3_stmt *pStmt){1070 int nVar = sqlite3_bind_parameter_count(pStmt);1071 int i;1072 for(i=1; i<=nVar; i++){1073 const char *zVar = sqlite3_bind_parameter_name(pStmt, i);1074 if( zVar==0 ) continue;1075#ifdef SQLITE_ENABLE_CARRAY1076 if( strcmp(zVar,"$carray_clr")==0 ){1077 static char *azColorNames[] = {1078 "azure", "black", "blue", "brown", "cyan", "fuchsia", "gold",1079 "gray", "green", "indigo", "khaki", "lime", "magenta", "maroon",1080 "navy", "olive", "orange", "pink", "purple", "red", "silver",1081 "tan", "teal", "violet", "white", "yellow"1082 };1083 sqlite3_carray_bind(pStmt,i,azColorNames,26,SQLITE_CARRAY_TEXT,0);1084 }else1085 if( strcmp(zVar,"$carray_primes")==0 ){1086 static int aPrimes[] = {1087 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47,1088 53, 59, 61, 67, 71, 73, 79, 83, 89, 971089 };1090 sqlite3_carray_bind(pStmt,i,aPrimes,26,SQLITE_CARRAY_INT32,0);1091 }else1092#endif1093 if( strncmp(zVar, "$int_", 5)==0 ){1094 sqlite3_bind_int(pStmt, i, atoi(&zVar[5]));1095 }else1096 if( strncmp(zVar, "$text_", 6)==0 ){1097 size_t szVar = strlen(zVar);1098 char *zBuf = sqlite3_malloc64( szVar-5 );1099 if( zBuf ){1100 memcpy(zBuf, &zVar[6], szVar-5);1101 sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8);1102 }1103 }1104 }1105}1106 1107/*1108** Run the SQL text1109*/1110static int runDbSql(1111 sqlite3 *db, /* Run SQL on this database connection */1112 const char *zSql, /* The SQL to be run */1113 unsigned int *pBtsFlags,1114 unsigned int dbOpt /* Default optimization flags */1115){1116 int rc;1117 sqlite3_stmt *pStmt;1118 int bCorrupt = 0;1119 while( isspace(zSql[0]&0x7f) ) zSql++;1120 if( zSql[0]==0 ) return SQLITE_OK;1121 if( eVerbosity>=4 ){1122 printf("RUNNING-SQL: [%s]\n", zSql);1123 fflush(stdout);1124 }1125 (*pBtsFlags) &= BTS_BADPRAGMA;1126 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);1127 if( rc==SQLITE_OK ){1128 int nRow = 0;1129 bindDebugParameters(pStmt);1130 while( (rc = sqlite3_step(pStmt))==SQLITE_ROW ){1131 nRow++;1132 if( eVerbosity>=4 ){1133 int j;1134 for(j=0; j<sqlite3_column_count(pStmt); j++){1135 if( j ) printf(",");1136 switch( sqlite3_column_type(pStmt, j) ){1137 case SQLITE_NULL: {1138 printf("NULL");1139 break;1140 }1141 case SQLITE_INTEGER:1142 case SQLITE_FLOAT: {1143 printf("%s", sqlite3_column_text(pStmt, j));1144 break;1145 }1146 case SQLITE_BLOB: {1147 int n = sqlite3_column_bytes(pStmt, j);1148 int i;1149 const unsigned char *a;1150 a = (const unsigned char*)sqlite3_column_blob(pStmt, j);1151 printf("x'");1152 for(i=0; i<n; i++){1153 printf("%02x", a[i]);1154 }1155 printf("'");1156 break;1157 }1158 case SQLITE_TEXT: {1159 int n = sqlite3_column_bytes(pStmt, j);1160 int i;1161 const unsigned char *a;1162 a = (const unsigned char*)sqlite3_column_blob(pStmt, j);1163 printf("'");1164 for(i=0; i<n; i++){1165 if( a[i]=='\'' ){1166 printf("''");1167 }else{1168 putchar(a[i]);1169 }1170 }1171 printf("'");1172 break;1173 }1174 } /* End switch() */1175 } /* End for() */1176 printf("\n");1177 fflush(stdout);1178 } /* End if( eVerbosity>=5 ) */1179 } /* End while( SQLITE_ROW */1180 if( rc==SQLITE_DONE ){1181 if( (*pBtsFlags)==BTS_SELECT1182 && !sqlite3_stmt_isexplain(pStmt)1183 && nRow>01184 ){1185 int iRow = 0;1186 sqlite3_reset(pStmt);1187 while( sqlite3_step(pStmt)==SQLITE_ROW ){1188 int iCnt = 0;1189 iRow++;1190 for(iCnt=0; iCnt<99999; iCnt++){1191 rc = fuzz_invariant(db, pStmt, iCnt, iRow, nRow,1192 &bCorrupt, eVerbosity, dbOpt);1193 if( rc==SQLITE_DONE ) break;1194 if( rc!=SQLITE_ERROR ) g.nInvariant++;1195 if( eVerbosity>0 ){1196 if( rc==SQLITE_OK ){1197 printf("invariant-check: ok\n");1198 }else if( rc==SQLITE_CORRUPT ){1199 printf("invariant-check: failed due to database corruption\n");1200 }