AryaWu/sqlite
0
1/*2** 2016-12-283**4** The author disclaims copyright to this source code. In place of5** a legal notice, here is a blessing:6**7** May you do good and not evil.8** May you find forgiveness for yourself and forgive others.9** May you share freely, never taking more than you give.10**11*************************************************************************12**13** This file implements "key-value" performance test for SQLite. The14** purpose is to compare the speed of SQLite for accessing large BLOBs15** versus reading those same BLOB values out of individual files in the16** filesystem.17**18** Run "kvtest" with no arguments for on-line help, or see comments below.19**20** HOW TO COMPILE:21**22** (1) Gather this source file and a recent SQLite3 amalgamation with its23** header into the working directory. You should have:24**25** kvtest.c >--- this file26** sqlite3.c \___ SQLite27** sqlite3.h / amlagamation & header28**29** (2) Run you compiler against the two C source code files.30**31** (a) On linux or mac:32**33** OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION"34** gcc -Os -I. $OPTS kvtest.c sqlite3.c -o kvtest35**36** The $OPTS options can be omitted. The $OPTS merely omit37** the need to link against -ldl and -lpthread, or whatever38** the equivalent libraries are called on your system.39**40** (b) Windows with MSVC:41**42** cl -I. kvtest.c sqlite3.c43**44** USAGE:45**46** (1) Create a test database by running "kvtest init" with appropriate47** options. See the help message for available options.48**49** (2) Construct the corresponding pile-of-files database on disk using50** the "kvtest export" command.51**52** (3) Run tests using "kvtest run" against either the SQLite database or53** the pile-of-files database and with appropriate options.54**55** For example:56**57** ./kvtest init x1.db --count 100000 --size 1000058** mkdir x159** ./kvtest export x1.db x160** ./kvtest run x1.db --count 10000 --max-id 100000061** ./kvtest run x1 --count 10000 --max-id 100000062*/63static const char zHelp[] = 64"Usage: kvtest COMMAND ARGS...\n"65"\n"66" kvtest init DBFILE --count N --size M --pagesize X\n"67"\n"68" Generate a new test database file named DBFILE containing N\n"69" BLOBs each of size M bytes. The page size of the new database\n"70" file will be X. Additional options:\n"71"\n"72" --variance V Randomly vary M by plus or minus V\n"73"\n"74" kvtest export DBFILE DIRECTORY [--tree]\n"75"\n"76" Export all the blobs in the kv table of DBFILE into separate\n"77" files in DIRECTORY. DIRECTORY is created if it does not previously\n"78" exist. If the --tree option is used, then the blobs are written\n"79" into a hierarchy of directories, using names like 00/00/00,\n"80" 00/00/01, 00/00/02, and so forth. Without the --tree option, all\n"81" files are in the top-level directory with names like 000000, 000001,\n"82" 000002, and so forth.\n"83"\n"84" kvtest stat DBFILE [options]\n"85"\n"86" Display summary information about DBFILE. Options:\n"87"\n"88" --vacuum Run VACUUM on the database file\n"89"\n"90" kvtest run DBFILE [options]\n"91"\n"92" Run a performance test. DBFILE can be either the name of a\n"93" database or a directory containing sample files. Options:\n"94"\n"95" --asc Read blobs in ascending order\n"96" --blob-api Use the BLOB API\n"97" --cache-size N Database cache size\n"98" --count N Read N blobs\n"99" --desc Read blobs in descending order\n"100" --fsync Synchronous file writes\n"101" --integrity-check Run \"PRAGMA integrity_check\" after test\n"102" --max-id N Maximum blob key to use\n"103" --mmap N Mmap as much as N bytes of DBFILE\n"104" --multitrans Each read or write in its own transaction\n"105" --nocheckpoint Omit the checkpoint on WAL mode writes\n"106" --nosync Set \"PRAGMA synchronous=OFF\"\n"107" --jmode MODE Set MODE journal mode prior to starting\n"108" --random Read blobs in a random order\n"109" --start N Start reading with this blob key\n"110" --stats Output operating stats before exiting\n"111" --update Do an overwrite test\n"112;113 114/* Reference resources used */115#include <stdio.h>116#include <stdlib.h>117#include <sys/types.h>118#include <sys/stat.h>119#include <assert.h>120#include <string.h>121#include "sqlite3.h"122 123#ifndef _WIN32124# include <unistd.h>125#else126 /* Provide Windows equivalent for the needed parts of unistd.h */127# include <direct.h>128# include <io.h>129# define R_OK 2130# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)131# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)132# define access _access133#endif134 135#if !defined(_MSC_VER)136# include <stdint.h>137#endif138 139/*140** The following macros are used to cast pointers to integers and141** integers to pointers. The way you do this varies from one compiler142** to the next, so we have developed the following set of #if statements143** to generate appropriate macros for a wide range of compilers.144**145** The correct "ANSI" way to do this is to use the intptr_t type.146** Unfortunately, that typedef is not available on all compilers, or147** if it is available, it requires an #include of specific headers148** that vary from one machine to the next.149**150** Ticket #3860: The llvm-gcc-4.2 compiler from Apple chokes on151** the ((void*)&((char*)0)[X]) construct. But MSVC chokes on ((void*)(X)).152** So we have to define the macros in different ways depending on the153** compiler.154*/155#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */156# define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X))157# define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(__PTRDIFF_TYPE__)(X))158#else159# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X))160# define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(intptr_t)(X))161#endif162 163/*164** Show the help text and quit.165*/166static void showHelp(void){167 fprintf(stdout, "%s", zHelp);168 exit(1);169}170 171/*172** Show an error message an quit.173*/174static void fatalError(const char *zFormat, ...){175 va_list ap;176 fprintf(stdout, "ERROR: ");177 va_start(ap, zFormat);178 vfprintf(stdout, zFormat, ap);179 va_end(ap);180 fprintf(stdout, "\n");181 exit(1);182}183 184/*185** Return the value of a hexadecimal digit. Return -1 if the input186** is not a hex digit.187*/188static int hexDigitValue(char c){189 if( c>='0' && c<='9' ) return c - '0';190 if( c>='a' && c<='f' ) return c - 'a' + 10;191 if( c>='A' && c<='F' ) return c - 'A' + 10;192 return -1;193}194 195/*196** Interpret zArg as an integer value, possibly with suffixes.197*/198static int integerValue(const char *zArg){199 int v = 0;200 static const struct { char *zSuffix; int iMult; } aMult[] = {201 { "KiB", 1024 },202 { "MiB", 1024*1024 },203 { "GiB", 1024*1024*1024 },204 { "KB", 1000 },205 { "MB", 1000000 },206 { "GB", 1000000000 },207 { "K", 1000 },208 { "M", 1000000 },209 { "G", 1000000000 },210 };211 int i;212 int isNeg = 0;213 if( zArg[0]=='-' ){214 isNeg = 1;215 zArg++;216 }else if( zArg[0]=='+' ){217 zArg++;218 }219 if( zArg[0]=='0' && zArg[1]=='x' ){220 int x;221 zArg += 2;222 while( (x = hexDigitValue(zArg[0]))>=0 ){223 v = (v<<4) + x;224 zArg++;225 }226 }else{227 while( zArg[0]>='0' && zArg[0]<='9' ){228 v = v*10 + zArg[0] - '0';229 zArg++;230 }231 }232 for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){233 if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){234 v *= aMult[i].iMult;235 break;236 }237 }238 return isNeg? -v : v;239}240 241 242/*243** Check the filesystem object zPath. Determine what it is:244**245** PATH_DIR A single directory holding many files246** PATH_TREE A directory hierarchy with files at the leaves247** PATH_DB An SQLite database248** PATH_NEXIST Does not exist249** PATH_OTHER Something else250**251** PATH_DIR means all of the separate files are grouped together252** into a single directory with names like 000000, 000001, 000002, and253** so forth. PATH_TREE means there is a hierarchy of directories so254** that no single directory has too many entries. The files have names255** like 00/00/00, 00/00/01, 00/00/02 and so forth. The decision between256** PATH_DIR and PATH_TREE is determined by the presence of a subdirectory257** named "00" at the top-level.258*/259#define PATH_DIR 1260#define PATH_TREE 2261#define PATH_DB 3262#define PATH_NEXIST 0263#define PATH_OTHER 99264static int pathType(const char *zPath){265 struct stat x;266 int rc;267 if( access(zPath,R_OK) ) return PATH_NEXIST;268 memset(&x, 0, sizeof(x));269 rc = stat(zPath, &x);270 if( rc<0 ) return PATH_OTHER;271 if( S_ISDIR(x.st_mode) ){272 char *zLayer1 = sqlite3_mprintf("%s/00", zPath);273 memset(&x, 0, sizeof(x));274 rc = stat(zLayer1, &x);275 sqlite3_free(zLayer1);276 if( rc<0 ) return PATH_DIR;277 if( S_ISDIR(x.st_mode) ) return PATH_TREE;278 return PATH_DIR;279 }280 if( (x.st_size%512)==0 ) return PATH_DB;281 return PATH_OTHER;282}283 284/*285** Return the size of a file in bytes. Or return -1 if the286** named object is not a regular file or does not exist.287*/288static sqlite3_int64 fileSize(const char *zPath){289 struct stat x;290 int rc;291 memset(&x, 0, sizeof(x));292 rc = stat(zPath, &x);293 if( rc<0 ) return -1;294 if( !S_ISREG(x.st_mode) ) return -1;295 return x.st_size;296}297 298/*299** A Pseudo-random number generator with a fixed seed. Use this so300** that the same sequence of "random" numbers are generated on each301** run, for repeatability.302*/303static unsigned int randInt(void){304 static unsigned int x = 0x333a13cd;305 static unsigned int y = 0xecb2adea;306 x = (x>>1) ^ ((1+~(x&1)) & 0xd0000001);307 y = y*1103515245 + 12345;308 return x^y;309}310 311/*312** Do database initialization.313*/314static int initMain(int argc, char **argv){315 char *zDb;316 int i, rc;317 int nCount = 1000;318 int sz = 10000;319 int iVariance = 0;320 int pgsz = 4096;321 sqlite3 *db;322 char *zSql;323 char *zErrMsg = 0;324 325 assert( strcmp(argv[1],"init")==0 );326 assert( argc>=3 );327 zDb = argv[2];328 for(i=3; i<argc; i++){329 char *z = argv[i];330 if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);331 if( z[1]=='-' ) z++;332 if( strcmp(z, "-count")==0 ){333 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);334 nCount = integerValue(argv[++i]);335 if( nCount<1 ) fatalError("the --count must be positive");336 continue;337 }338 if( strcmp(z, "-size")==0 ){339 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);340 sz = integerValue(argv[++i]);341 if( sz<1 ) fatalError("the --size must be positive");342 continue;343 }344 if( strcmp(z, "-variance")==0 ){345 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);346 iVariance = integerValue(argv[++i]);347 continue;348 }349 if( strcmp(z, "-pagesize")==0 ){350 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);351 pgsz = integerValue(argv[++i]);352 if( pgsz<512 || pgsz>65536 || ((pgsz-1)&pgsz)!=0 ){353 fatalError("the --pagesize must be power of 2 between 512 and 65536");354 }355 continue;356 }357 fatalError("unknown option: \"%s\"", argv[i]);358 }359 rc = sqlite3_open(zDb, &db);360 if( rc ){361 fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));362 }363 zSql = sqlite3_mprintf(364 "DROP TABLE IF EXISTS kv;\n"365 "PRAGMA page_size=%d;\n"366 "VACUUM;\n"367 "BEGIN;\n"368 "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n"369 "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)"370 " INSERT INTO kv(k,v) SELECT x, randomblob(%d+(random()%%(%d))) FROM c;\n"371 "COMMIT;\n",372 pgsz, nCount, sz, iVariance+1373 );374 rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);375 if( rc ) fatalError("database create failed: %s", zErrMsg);376 sqlite3_free(zSql);377 sqlite3_close(db);378 return 0;379}380 381/*382** Analyze an existing database file. Report its content.383*/384static int statMain(int argc, char **argv){385 char *zDb;386 int i, rc;387 sqlite3 *db;388 char *zSql;389 sqlite3_stmt *pStmt;390 int doVacuum = 0;391 392 assert( strcmp(argv[1],"stat")==0 );393 assert( argc>=3 );394 zDb = argv[2];395 for(i=3; i<argc; i++){396 char *z = argv[i];397 if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);398 if( z[1]=='-' ) z++;399 if( strcmp(z, "-vacuum")==0 ){400 doVacuum = 1;401 continue;402 }403 fatalError("unknown option: \"%s\"", argv[i]);404 }405 rc = sqlite3_open(zDb, &db);406 if( rc ){407 fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));408 }409 if( doVacuum ){410 printf("Vacuuming...."); fflush(stdout);411 sqlite3_exec(db, "VACUUM", 0, 0, 0);412 printf(" done\n");413 }414 zSql = sqlite3_mprintf(415 "SELECT count(*), min(length(v)), max(length(v)), avg(length(v))"416 " FROM kv"417 );418 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);419 if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));420 sqlite3_free(zSql);421 if( sqlite3_step(pStmt)==SQLITE_ROW ){422 printf("Number of entries: %8d\n", sqlite3_column_int(pStmt, 0));423 printf("Average value size: %8d\n", sqlite3_column_int(pStmt, 3));424 printf("Minimum value size: %8d\n", sqlite3_column_int(pStmt, 1));425 printf("Maximum value size: %8d\n", sqlite3_column_int(pStmt, 2));426 }else{427 printf("No rows\n");428 }429 sqlite3_finalize(pStmt);430 zSql = sqlite3_mprintf("PRAGMA page_size");431 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);432 if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));433 sqlite3_free(zSql);434 if( sqlite3_step(pStmt)==SQLITE_ROW ){435 printf("Page-size: %8d\n", sqlite3_column_int(pStmt, 0));436 }437 sqlite3_finalize(pStmt);438 zSql = sqlite3_mprintf("PRAGMA page_count");439 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);440 if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));441 sqlite3_free(zSql);442 if( sqlite3_step(pStmt)==SQLITE_ROW ){443 printf("Page-count: %8d\n", sqlite3_column_int(pStmt, 0));444 }445 sqlite3_finalize(pStmt);446 zSql = sqlite3_mprintf("PRAGMA freelist_count");447 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);448 if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));449 sqlite3_free(zSql);450 if( sqlite3_step(pStmt)==SQLITE_ROW ){451 printf("Freelist-count: %8d\n", sqlite3_column_int(pStmt, 0));452 }453 sqlite3_finalize(pStmt);454 rc = sqlite3_prepare_v2(db, "PRAGMA integrity_check(10)", -1, &pStmt, 0);455 if( rc ) fatalError("cannot prepare integrity check: %s", sqlite3_errmsg(db));456 while( sqlite3_step(pStmt)==SQLITE_ROW ){457 printf("Integrity-check: %s\n", sqlite3_column_text(pStmt, 0));458 }459 sqlite3_finalize(pStmt);460 sqlite3_close(db);461 return 0;462}463 464/*465** remember(V,PTR)466**467** Return the integer value V. Also save the value of V in a468** C-language variable whose address is PTR.469*/470static void rememberFunc(471 sqlite3_context *pCtx,472 int argc,473 sqlite3_value **argv474){475 sqlite3_int64 v;476 sqlite3_int64 ptr;477 assert( argc==2 );478 v = sqlite3_value_int64(argv[0]);479 ptr = sqlite3_value_int64(argv[1]);480 *(sqlite3_int64*)SQLITE_INT_TO_PTR(ptr) = v;481 sqlite3_result_int64(pCtx, v);482}483 484/*485** Make sure a directory named zDir exists.486*/487static void kvtest_mkdir(const char *zDir){488#if defined(_WIN32)489 (void)mkdir(zDir);490#else491 (void)mkdir(zDir, 0755);492#endif493}494 495/*496** Export the kv table to individual files in the filesystem497*/498static int exportMain(int argc, char **argv){499 char *zDb;500 char *zDir;501 sqlite3 *db;502 sqlite3_stmt *pStmt;503 int rc;504 int ePathType;505 int nFN;506 char *zFN;507 char *zTail;508 size_t nWrote;509 int i;510 511 assert( strcmp(argv[1],"export")==0 );512 assert( argc>=3 );513 if( argc<4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY [OPTIONS]");514 zDb = argv[2];515 zDir = argv[3];516 kvtest_mkdir(zDir);517 for(i=4; i<argc; i++){518 const char *z = argv[i];519 if( z[0]=='-' && z[1]=='-' ) z++;520 if( strcmp(z,"-tree")==0 ){521 zFN = sqlite3_mprintf("%s/00", zDir);522 kvtest_mkdir(zFN);523 sqlite3_free(zFN);524 continue;525 }526 fatalError("unknown argument: \"%s\"\n", argv[i]);527 }528 ePathType = pathType(zDir);529 if( ePathType!=PATH_DIR && ePathType!=PATH_TREE ){530 fatalError("object \"%s\" is not a directory", zDir);531 }532 rc = sqlite3_open(zDb, &db);533 if( rc ){534 fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));535 }536 rc = sqlite3_prepare_v2(db, "SELECT k, v FROM kv ORDER BY k", -1, &pStmt, 0);537 if( rc ){538 fatalError("prepare_v2 failed: %s\n", sqlite3_errmsg(db));539 }540 nFN = (int)strlen(zDir);541 zFN = sqlite3_mprintf("%s/00/00/00.extra---------------------", zDir);542 if( zFN==0 ){543 fatalError("malloc failed\n");544 }545 zTail = zFN + nFN + 1;546 while( sqlite3_step(pStmt)==SQLITE_ROW ){547 int iKey = sqlite3_column_int(pStmt, 0);548 sqlite3_int64 nData = sqlite3_column_bytes(pStmt, 1);549 const void *pData = sqlite3_column_blob(pStmt, 1);550 FILE *out;551 if( ePathType==PATH_DIR ){552 sqlite3_snprintf(20, zTail, "%06d", iKey);553 }else{554 sqlite3_snprintf(20, zTail, "%02d", iKey/10000);555 kvtest_mkdir(zFN);556 sqlite3_snprintf(20, zTail, "%02d/%02d", iKey/10000, (iKey/100)%100);557 kvtest_mkdir(zFN);558 sqlite3_snprintf(20, zTail, "%02d/%02d/%02d",559 iKey/10000, (iKey/100)%100, iKey%100);560 }561 out = fopen(zFN, "wb"); 562 nWrote = fwrite(pData, 1, (size_t)nData, out);563 fclose(out);564 printf("\r%s ", zTail); fflush(stdout);565 if( nWrote!=(size_t)nData ){566 fatalError("Wrote only %d of %d bytes to %s\n",567 (int)nWrote, nData, zFN);568 }569 }570 sqlite3_finalize(pStmt);571 sqlite3_close(db);572 sqlite3_free(zFN);573 printf("\n");574 return 0;575}576 577/*578** Read the content of file zName into memory obtained from sqlite3_malloc64()579** and return a pointer to the buffer. The caller is responsible for freeing 580** the memory. 581**582** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes583** read.584**585** For convenience, a nul-terminator byte is always appended to the data read586** from the file before the buffer is returned. This byte is not included in587** the final value of (*pnByte), if applicable.588**589** NULL is returned if any error is encountered. The final value of *pnByte590** is undefined in this case.591*/592static unsigned char *readFile(const char *zName, sqlite3_int64 *pnByte){593 FILE *in; /* FILE from which to read content of zName */594 sqlite3_int64 nIn; /* Size of zName in bytes */595 size_t nRead; /* Number of bytes actually read */596 unsigned char *pBuf; /* Content read from disk */597 598 nIn = fileSize(zName);599 if( nIn<0 ) return 0;600 in = fopen(zName, "rb");601 if( in==0 ) return 0;602 pBuf = sqlite3_malloc64( nIn );603 if( pBuf==0 ) return 0;604 nRead = fread(pBuf, (size_t)nIn, 1, in);605 fclose(in);606 if( nRead!=1 ){607 sqlite3_free(pBuf);608 return 0;609 }610 if( pnByte ) *pnByte = nIn;611 return pBuf;612}613 614/*615** Overwrite a file with randomness. Do not change the size of the616** file.617*/618static void updateFile(const char *zName, sqlite3_int64 *pnByte, int doFsync){619 FILE *out; /* FILE from which to read content of zName */620 sqlite3_int64 sz; /* Size of zName in bytes */621 size_t nWritten; /* Number of bytes actually read */622 unsigned char *pBuf; /* Content to store on disk */623 const char *zMode = "wb"; /* Mode for fopen() */624 625 sz = fileSize(zName);626 if( sz<0 ){627 fatalError("No such file: \"%s\"", zName);628 }629 *pnByte = sz;630 if( sz==0 ) return;631 pBuf = sqlite3_malloc64( sz );632 if( pBuf==0 ){633 fatalError("Cannot allocate %lld bytes\n", sz);634 }635 sqlite3_randomness((int)sz, pBuf); 636#if defined(_WIN32)637 if( doFsync ) zMode = "wbc";638#endif639 out = fopen(zName, zMode);640 if( out==0 ){641 fatalError("Cannot open \"%s\" for writing\n", zName);642 }643 nWritten = fwrite(pBuf, 1, (size_t)sz, out);644 if( doFsync ){645#if defined(_WIN32)646 fflush(out);647#else648 fsync(fileno(out));649#endif650 }651 fclose(out);652 if( nWritten!=(size_t)sz ){653 fatalError("Wrote only %d of %d bytes to \"%s\"\n",654 (int)nWritten, (int)sz, zName);655 }656 sqlite3_free(pBuf);657}658 659/*660** Return the current time in milliseconds since the beginning of661** the Julian epoch.662*/663static sqlite3_int64 timeOfDay(void){664 static sqlite3_vfs *clockVfs = 0;665 sqlite3_int64 t;666 if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0);667 if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){668 clockVfs->xCurrentTimeInt64(clockVfs, &t);669 }else{670 double r;671 clockVfs->xCurrentTime(clockVfs, &r);672 t = (sqlite3_int64)(r*86400000.0);673 }674 return t;675}676 677#ifdef __linux__678/*679** Attempt to display I/O stats on Linux using /proc/PID/io680*/681static void displayLinuxIoStats(FILE *out){682 FILE *in;683 char z[200];684 sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid());685 in = fopen(z, "rb");686 if( in==0 ) return;687 while( fgets(z, sizeof(z), in)!=0 ){688 static const struct {689 const char *zPattern;690 const char *zDesc;691 } aTrans[] = {692 { "rchar: ", "Bytes received by read():" },693 { "wchar: ", "Bytes sent to write():" },694 { "syscr: ", "Read() system calls:" },695 { "syscw: ", "Write() system calls:" },696 { "read_bytes: ", "Bytes read from storage:" },697 { "write_bytes: ", "Bytes written to storage:" },698 { "cancelled_write_bytes: ", "Cancelled write bytes:" },699 };700 int i;701 for(i=0; i<sizeof(aTrans)/sizeof(aTrans[0]); i++){702 int n = (int)strlen(aTrans[i].zPattern);703 if( strncmp(aTrans[i].zPattern, z, n)==0 ){704 fprintf(out, "%-36s %s", aTrans[i].zDesc, &z[n]);705 break;706 }707 }708 }709 fclose(in);710}711#endif712 713/*714** Display memory stats.715*/716static int display_stats(717 sqlite3 *db, /* Database to query */718 int bReset /* True to reset SQLite stats */719){720 int iCur;721 int iHiwtr;722 FILE *out = stdout;723 724 fprintf(out, "\n");725 726 iHiwtr = iCur = -1;727 sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset);728 fprintf(out,729 "Memory Used: %d (max %d) bytes\n",730 iCur, iHiwtr);731 iHiwtr = iCur = -1;732 sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset);733 fprintf(out, "Number of Outstanding Allocations: %d (max %d)\n",734 iCur, iHiwtr);735 iHiwtr = iCur = -1;736 sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset);737 fprintf(out,738 "Number of Pcache Pages Used: %d (max %d) pages\n",739 iCur, iHiwtr);740 iHiwtr = iCur = -1;741 sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset);742 fprintf(out,743 "Number of Pcache Overflow Bytes: %d (max %d) bytes\n",744 iCur, iHiwtr);745 iHiwtr = iCur = -1;746 sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset);747 fprintf(out, "Largest Allocation: %d bytes\n",748 iHiwtr);749 iHiwtr = iCur = -1;750 sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset);751 fprintf(out, "Largest Pcache Allocation: %d bytes\n",752 iHiwtr);753 754 iHiwtr = iCur = -1;755 sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset);756 fprintf(out, "Pager Heap Usage: %d bytes\n",757 iCur);758 iHiwtr = iCur = -1;759 sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1);760 fprintf(out, "Page cache hits: %d\n", iCur);761 iHiwtr = iCur = -1;762 sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1);763 fprintf(out, "Page cache misses: %d\n", iCur);764 iHiwtr = iCur = -1;765 sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1);766 fprintf(out, "Page cache writes: %d\n", iCur);767 iHiwtr = iCur = -1;768 769#ifdef __linux__770 displayLinuxIoStats(out);771#endif772 773 return 0;774}775 776/* Blob access order */777#define ORDER_ASC 1778#define ORDER_DESC 2779#define ORDER_RANDOM 3780 781 782/*783** Run a performance test784*/785static int runMain(int argc, char **argv){786 int eType; /* Is zDb a database or a directory? */787 char *zDb; /* Database or directory name */788 int i; /* Loop counter */789 int rc; /* Return code from SQLite calls */790 int nCount = 1000; /* Number of blob fetch operations */791 int nExtra = 0; /* Extra cycles */792 int iKey = 1; /* Next blob key */793 int iMax = 0; /* Largest allowed key */794 int iPagesize = 0; /* Database page size */795 int iCache = 1000; /* Database cache size in kibibytes */796 int bBlobApi = 0; /* Use the incremental blob I/O API */797 int bStats = 0; /* Print stats before exiting */798 int eOrder = ORDER_ASC; /* Access order */799 int isUpdateTest = 0; /* Do in-place updates rather than reads */800 int doIntegrityCk = 0; /* Run PRAGMA integrity_check after the test */801 int noSync = 0; /* Disable synchronous mode */802 int doFsync = 0; /* Update disk files synchronously */803 int doMultiTrans = 0; /* Each operation in its own transaction */804 int noCheckpoint = 0; /* Omit the checkpoint in WAL mode */805 sqlite3 *db = 0; /* Database connection */806 sqlite3_stmt *pStmt = 0; /* Prepared statement for SQL access */807 sqlite3_blob *pBlob = 0; /* Handle for incremental Blob I/O */808 sqlite3_int64 tmStart; /* Start time */809 sqlite3_int64 tmElapsed; /* Elapsed time */810 int mmapSize = 0; /* --mmap N argument */811 sqlite3_int64 nData = 0; /* Bytes of data */812 sqlite3_int64 nTotal = 0; /* Total data read */813 unsigned char *pData = 0; /* Content of the blob */814 sqlite3_int64 nAlloc = 0; /* Space allocated for pData[] */815 const char *zJMode = 0; /* Journal mode */816 817 818 assert( strcmp(argv[1],"run")==0 );819 assert( argc>=3 );820 zDb = argv[2];821 eType = pathType(zDb);822 if( eType==PATH_OTHER ) fatalError("unknown object type: \"%s\"", zDb);823 if( eType==PATH_NEXIST ) fatalError("object does not exist: \"%s\"", zDb);824 for(i=3; i<argc; i++){825 char *z = argv[i];826 if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);827 if( z[1]=='-' ) z++;828 if( strcmp(z, "-asc")==0 ){829 eOrder = ORDER_ASC;830 continue;831 }832 if( strcmp(z, "-blob-api")==0 ){833 bBlobApi = 1;834 continue;835 }836 if( strcmp(z, "-cache-size")==0 ){837 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);838 iCache = integerValue(argv[++i]);839 continue;840 }841 if( strcmp(z, "-count")==0 ){842 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);843 nCount = integerValue(argv[++i]);844 if( nCount<1 ) fatalError("the --count must be positive");845 continue;846 }847 if( strcmp(z, "-desc")==0 ){848 eOrder = ORDER_DESC;849 continue;850 }851 if( strcmp(z, "-fsync")==0 ){852 doFsync = 1;853 continue;854 }855 if( strcmp(z, "-integrity-check")==0 ){856 doIntegrityCk = 1;857 continue;858 }859 if( strcmp(z, "-jmode")==0 ){860 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);861 zJMode = argv[++i];862 continue;863 }864 if( strcmp(z, "-mmap")==0 ){865 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);866 mmapSize = integerValue(argv[++i]);867 if( nCount<0 ) fatalError("the --mmap must be non-negative");868 continue;869 }870 if( strcmp(z, "-max-id")==0 ){871 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);872 iMax = integerValue(argv[++i]);873 continue;874 }875 if( strcmp(z, "-multitrans")==0 ){876 doMultiTrans = 1;877 continue;878 }879 if( strcmp(z, "-nocheckpoint")==0 ){880 noCheckpoint = 1;881 continue;882 }883 if( strcmp(z, "-nosync")==0 ){884 noSync = 1;885 continue;886 }887 if( strcmp(z, "-random")==0 ){888 eOrder = ORDER_RANDOM;889 continue;890 }891 if( strcmp(z, "-start")==0 ){892 if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);893 iKey = integerValue(argv[++i]);894 if( iKey<1 ) fatalError("the --start must be positive");895 continue;896 }897 if( strcmp(z, "-stats")==0 ){898 bStats = 1;899 continue;900 }901 if( strcmp(z, "-update")==0 ){902 isUpdateTest = 1;903 continue;904 }905 fatalError("unknown option: \"%s\"", argv[i]);906 }907 if( eType==PATH_DB ){908 /* Recover any prior crashes prior to starting the timer */909 sqlite3_open(zDb, &db);910 sqlite3_exec(db, "SELECT rowid FROM sqlite_schema LIMIT 1", 0, 0, 0);911 sqlite3_close(db);912 db = 0;913 }914 tmStart = timeOfDay();915 if( eType==PATH_DB ){916 char *zSql;917 rc = sqlite3_open(zDb, &db);918 if( rc ){919 fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));920 }921 zSql = sqlite3_mprintf("PRAGMA mmap_size=%d", mmapSize);922 sqlite3_exec(db, zSql, 0, 0, 0);923 sqlite3_free(zSql);924 zSql = sqlite3_mprintf("PRAGMA cache_size=%d", iCache);925 sqlite3_exec(db, zSql, 0, 0, 0);926 sqlite3_free(zSql);927 if( noSync ){928 sqlite3_exec(db, "PRAGMA synchronous=OFF", 0, 0, 0);929 }930 pStmt = 0;931 sqlite3_prepare_v2(db, "PRAGMA page_size", -1, &pStmt, 0);932 if( sqlite3_step(pStmt)==SQLITE_ROW ){933 iPagesize = sqlite3_column_int(pStmt, 0);934 }935 sqlite3_finalize(pStmt);936 sqlite3_prepare_v2(db, "PRAGMA cache_size", -1, &pStmt, 0);937 if( sqlite3_step(pStmt)==SQLITE_ROW ){938 iCache = sqlite3_column_int(pStmt, 0);939 }else{940 iCache = 0;941 }942 sqlite3_finalize(pStmt);943 pStmt = 0;944 if( zJMode ){945 zSql = sqlite3_mprintf("PRAGMA journal_mode=%Q", zJMode);946 sqlite3_exec(db, zSql, 0, 0, 0);947 sqlite3_free(zSql);948 if( noCheckpoint ){949 sqlite3_exec(db, "PRAGMA wal_autocheckpoint=0", 0, 0, 0);950 }951 }952 sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &pStmt, 0);953 if( sqlite3_step(pStmt)==SQLITE_ROW ){954 zJMode = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));955 }else{956 zJMode = "???";957 }958 sqlite3_finalize(pStmt);959 if( iMax<=0 ){960 sqlite3_prepare_v2(db, "SELECT max(k) FROM kv", -1, &pStmt, 0);961 if( sqlite3_step(pStmt)==SQLITE_ROW ){962 iMax = sqlite3_column_int(pStmt, 0);963 }964 sqlite3_finalize(pStmt);965 }966 pStmt = 0;967 if( !doMultiTrans ) sqlite3_exec(db, "BEGIN", 0, 0, 0);968 }969 if( iMax<=0 ) iMax = 1000;970 for(i=0; i<nCount; i++){971 if( eType==PATH_DIR || eType==PATH_TREE ){972 /* CASE 1: Reading or writing blobs out of separate files */973 char *zKey;974 if( eType==PATH_DIR ){975 zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);976 }else{977 zKey = sqlite3_mprintf("%s/%02d/%02d/%02d", zDb, iKey/10000,978 (iKey/100)%100, iKey%100);979 }980 nData = 0;981 if( isUpdateTest ){982 updateFile(zKey, &nData, doFsync);983 }else{984 pData = readFile(zKey, &nData);985 sqlite3_free(pData);986 }987 sqlite3_free(zKey);988 }else if( bBlobApi ){989 /* CASE 2: Reading from database using the incremental BLOB I/O API */990 if( pBlob==0 ){991 rc = sqlite3_blob_open(db, "main", "kv", "v", iKey,992 isUpdateTest, &pBlob);993 if( rc ){994 fatalError("could not open sqlite3_blob handle: %s",995 sqlite3_errmsg(db));996 }997 }else{998 rc = sqlite3_blob_reopen(pBlob, iKey);999 }1000 if( rc==SQLITE_OK ){1001 nData = sqlite3_blob_bytes(pBlob);1002 if( nAlloc<nData+1 ){1003 nAlloc = nData+100;1004 pData = sqlite3_realloc64(pData, nAlloc);1005 }1006 if( pData==0 ) fatalError("cannot allocate %d bytes", nData+1);1007 if( isUpdateTest ){1008 sqlite3_randomness((int)nData, pData);1009 rc = sqlite3_blob_write(pBlob, pData, (int)nData, 0);1010 if( rc!=SQLITE_OK ){1011 fatalError("could not write the blob at %d: %s", iKey,1012 sqlite3_errmsg(db));1013 }1014 }else{1015 rc = sqlite3_blob_read(pBlob, pData, (int)nData, 0);1016 if( rc!=SQLITE_OK ){1017 fatalError("could not read the blob at %d: %s", iKey,1018 sqlite3_errmsg(db));1019 }1020 }1021 }1022 }else{1023 /* CASE 3: Reading from database using SQL */1024 if( pStmt==0 ){1025 if( isUpdateTest ){1026 sqlite3_create_function(db, "remember", 2, SQLITE_UTF8, 0,1027 rememberFunc, 0, 0);1028 1029 rc = sqlite3_prepare_v2(db, 1030 "UPDATE kv SET v=randomblob(remember(length(v),?2))"1031 " WHERE k=?1", -1, &pStmt, 0);1032 sqlite3_bind_int64(pStmt, 2, SQLITE_PTR_TO_INT(&nData));1033 }else{1034 rc = sqlite3_prepare_v2(db, 1035 "SELECT v FROM kv WHERE k=?1", -1, &pStmt, 0);1036 }1037 if( rc ){1038 fatalError("cannot prepare query: %s", sqlite3_errmsg(db));1039 }1040 }else{1041 sqlite3_reset(pStmt);1042 }1043 sqlite3_bind_int(pStmt, 1, iKey);1044 nData = 0;1045 rc = sqlite3_step(pStmt);1046 if( rc==SQLITE_ROW ){1047 nData = sqlite3_column_bytes(pStmt, 0);1048 pData = (unsigned char*)sqlite3_column_blob(pStmt, 0);1049 }1050 }1051 if( eOrder==ORDER_ASC ){1052 iKey++;1053 if( iKey>iMax ) iKey = 1;1054 }else if( eOrder==ORDER_DESC ){1055 iKey--;1056 if( iKey<=0 ) iKey = iMax;1057 }else{1058 iKey = (randInt()%iMax)+1;1059 }1060 nTotal += nData;1061 if( nData==0 ){ nCount++; nExtra++; }1062 }1063 if( nAlloc ) sqlite3_free(pData);1064 if( pStmt ) sqlite3_finalize(pStmt);1065 if( pBlob ) sqlite3_blob_close(pBlob);1066 if( bStats ){1067 display_stats(db, 0);1068 }1069 if( db ){1070 if( !doMultiTrans ) sqlite3_exec(db, "COMMIT", 0, 0, 0);1071 if( !noCheckpoint ){1072 sqlite3_close(db);1073 db = 0;1074 }1075 }1076 tmElapsed = timeOfDay() - tmStart;1077 if( db && noCheckpoint ){1078 sqlite3_close(db);1079 db = 0;1080 }1081 if( nExtra ){1082 printf("%d cycles due to %d misses\n", nCount, nExtra);1083 }1084 if( eType==PATH_DB ){1085 printf("SQLite version: %s\n", sqlite3_libversion());1086 if( doIntegrityCk ){1087 sqlite3_open(zDb, &db);1088 sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &pStmt, 0);1089 while( sqlite3_step(pStmt)==SQLITE_ROW ){1090 printf("integrity-check: %s\n", sqlite3_column_text(pStmt, 0));1091 }1092 sqlite3_finalize(pStmt);1093 sqlite3_close(db);1094 db = 0;1095 }1096 }1097 printf("--count %d --max-id %d", nCount-nExtra, iMax);1098 switch( eOrder ){1099 case ORDER_RANDOM: printf(" --random\n"); break;1100 case ORDER_DESC: printf(" --desc\n"); break;1101 default: printf(" --asc\n"); break;1102 }1103 if( eType==PATH_DB ){1104 printf("--cache-size %d --jmode %s\n", iCache, zJMode);1105 printf("--mmap %d%s\n", mmapSize, bBlobApi ? " --blob-api" : "");1106 if( noSync ) printf("--nosync\n");1107 }1108 if( iPagesize ) printf("Database page size: %d\n", iPagesize);1109 printf("Total elapsed time: %.3f\n", tmElapsed/1000.0);1110 if( isUpdateTest ){1111 printf("Microseconds per BLOB write: %.3f\n", tmElapsed*1000.0/nCount);1112 printf("Content write rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));1113 }else{1114 printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount);1115 printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));1116 }1117 return 0;1118}1119 1120 1121int main(int argc, char **argv){1122 if( argc<3 ) showHelp();1123 if( strcmp(argv[1],"init")==0 ){1124 return initMain(argc, argv);1125 }1126 if( strcmp(argv[1],"export")==0 ){1127 return exportMain(argc, argv);1128 }1129 if( strcmp(argv[1],"run")==0 ){1130 return runMain(argc, argv);1131 }1132 if( strcmp(argv[1],"stat")==0 ){1133 return statMain(argc, argv);1134 }1135 showHelp();1136 return 0;1137}1138